{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f72fa2d8-1946-43e4-ac8b-8de1060f03d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "import random\n",
    "import numpy as np\n",
    "import torch.nn as nn\n",
    "import matplotlib.pyplot as plt\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2002b900-380e-4bed-8a57-2910968f0f7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dir= 'F:/data/mnist/test/mnist_test.csv'\n",
    "train_ratio, valid_ratio, test_ratio, k= 0.8, 0.1, 0.1, 0\n",
    "lr, device, patience, pt_folder, pt_file4vqgan, pt_file4disc, pt_file4tfm, epoch_num, in_dim, out_dim, batch_size= 5e-3, 'cuda:0', 80, './', 'vqgan.pt', 'disc.pt', 'tfm.pt', 1000, 784, 128, 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2160af73-2e0d-45bc-ab42-8819facfb32d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_mnist_data(dir):\n",
    "    with open(dir, encoding= 'utf-8-sig') as f:\n",
    "        lines= f.readlines()\n",
    "    label_pic_ts= torch.from_numpy(np.asfarray([line.rstrip().split(',') for line in lines])).float()\n",
    "    return label_pic_ts[:, 1:]/ 255.0, label_pic_ts[:, 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8244cdcf-2e29-429d-9f5e-3738f496365f",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        # (B, 1, 28, 28)>> (B, 32, 14, 14)>> (B, 16, 7, 7)\n",
    "        self.net= nn.Sequential(nn.Conv2d(in_channels= 1, out_channels= 32, kernel_size= 4, stride= 2, padding= 1), nn.ReLU(), nn.Dropout(0.3), nn.Conv2d(32, 16, 4, 2, 1), nn.ReLU())\n",
    "        self.set_para()\n",
    "    def set_para(self):\n",
    "        nn.init.xavier_normal_(self.net[0].weight, nn.init.calculate_gain('relu'))\n",
    "        nn.init.xavier_normal_(self.net[3].weight, nn.init.calculate_gain('relu'))\n",
    "    def forward(self, x):\n",
    "        return self.net(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "72eb2056-c2ea-4683-a1df-948feefbd60c",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.net= nn.Sequential(nn.ConvTranspose2d(16, 32, 4, 2, 1), nn.ReLU(), nn.ConvTranspose2d(32, 1, 4, 2, 1), nn.Sigmoid())\n",
    "        self.set_para()\n",
    "    def set_para(self):\n",
    "        nn.init.xavier_normal_(self.net[0].weight, nn.init.calculate_gain('relu'))\n",
    "        nn.init.xavier_normal_(self.net[2].weight, nn.init.calculate_gain('sigmoid'))\n",
    "    def forward(self, x):\n",
    "        return self.net(x)     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ed3593cb-af8f-4fa0-bc46-fc94c3c577a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Codebook(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Codebook, self).__init__()\n",
    "        self.num_codebook_vectors, self.latent_dim= 20, 16\n",
    "        self.beta= 0.25\n",
    "        self.embedding= nn.Embedding(self.num_codebook_vectors, self.latent_dim)\n",
    "        self.set_para()\n",
    "    def set_para(self):\n",
    "        nn.init.xavier_normal_(self.embedding.weight)\n",
    "    def forward(self, z):\n",
    "        # (B, 16, 7, 7)>> (B, 7, 7, 16)\n",
    "        z= z.permute(0, 2, 3, 1).contiguous()\n",
    "        # (B* 7* 7, 16)\n",
    "        z_flattened= z.view(-1, self.latent_dim)\n",
    "        # (B* 7* 7, 1)+ (20, )- (B* 7* 7, 20)= (49B, 20)\n",
    "        d= torch.sum(z_flattened** 2, dim= 1, keepdim= True)+ torch.sum(self.embedding.weight** 2, dim= 1)- 2* torch.matmul(z_flattened, self.embedding.weight.T)\n",
    "        # (49B,)\n",
    "        min_encoding_indices= torch.argmin(d, dim= 1)\n",
    "        # (49B, 16)>> (B, 7, 7, 16)\n",
    "        z_q= self.embedding(min_encoding_indices).view(z.shape)\n",
    "        loss= torch.mean((z_q.detach()- z)** 2)+ self.beta* torch.mean((z_q- z.detach())** 2)\n",
    "        z_q= z+ (z_q- z).detach()\n",
    "        # (B, 16, 7, 7)\n",
    "        z_q= z_q.permute(0, 3, 1, 2)\n",
    "        # (B, 16, 7, 7), (49B,). \n",
    "        return z_q, min_encoding_indices, loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3d85e8e1-1c5e-4281-ab00-f8ad7399807a",
   "metadata": {},
   "outputs": [],
   "source": [
    "class VQGAN(nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super(VQGAN, self).__init__()\n",
    "        self.device= 'cuda:0'\n",
    "        self.encoder= Encoder().to(self.device)\n",
    "        self.decoder= Decoder().to(self.device)\n",
    "        self.codebook= Codebook().to(self.device)\n",
    "        self.quant_conv= nn.Conv2d(16, 16, 1).to(self.device)\n",
    "        self.post_quant_conv= nn.Conv2d(16, 16, 1).to(self.device)\n",
    "        self.set_para()\n",
    "        \n",
    "    def set_para(self):\n",
    "        nn.init.xavier_normal_(self.quant_conv.weight)\n",
    "        nn.init.xavier_normal_(self.post_quant_conv.weight)\n",
    "\n",
    "    def mse_loss(self, label, pred):\n",
    "        return ((label- pred)** 2).mean()\n",
    "\n",
    "    def forward(self, x):\n",
    "        encoded_images= self.encoder(x)\n",
    "        quant_conv_encoded_images= self.quant_conv(encoded_images)\n",
    "        codebook_mapping, codebook_indices, q_loss= self.codebook(quant_conv_encoded_images)\n",
    "        post_quant_conv_mapping= self.post_quant_conv(codebook_mapping)\n",
    "        decoded_images= self.decoder(post_quant_conv_mapping)\n",
    "        return decoded_images, codebook_indices, q_loss\n",
    "\n",
    "    def encode(self, x):\n",
    "        # (B, 16, 7, 7)\n",
    "        encoded_images= self.encoder(x)\n",
    "        # (B, 16, 1, 1)\n",
    "        quant_conv_encoded_images= self.quant_conv(encoded_images)\n",
    "        # \n",
    "        codebook_mapping, codebook_indices, q_loss= self.codebook(quant_conv_encoded_images)\n",
    "        return codebook_mapping, codebook_indices, q_loss\n",
    "\n",
    "    def decode(self, z):\n",
    "        post_quant_conv_mapping= self.post_quant_conv(z)\n",
    "        decoded_images= self.decoder(post_quant_conv_mapping)\n",
    "        return decoded_images\n",
    "\n",
    "    def calculate_lambda(self, recon_loss, gan_loss):\n",
    "        last_layer= self.decoder.net[-2]\n",
    "        last_layer_weight= last_layer.weight\n",
    "        recon_loss_grads= torch.autograd.grad(recon_loss, last_layer_weight, retain_graph= True)[0]\n",
    "        gan_loss_grads= torch.autograd.grad(gan_loss, last_layer_weight, retain_graph= True)[0]\n",
    "        λ= torch.norm(recon_loss_grads)/ (torch.norm(gan_loss_grads)+ 1e-10)\n",
    "        return 0.8* torch.clamp(λ, 0, 1e4).detach()\n",
    "\n",
    "    @staticmethod\n",
    "    def adopt_weight(disc_factor, i, threshold, value= 0.):\n",
    "        if i< threshold: disc_factor= value\n",
    "        return disc_factor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1d02203e-a8ac-441a-9757-68414ab1655e",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Discriminator(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.cv= nn.Sequential(nn.Conv2d(in_channels= 1, out_channels= 32, kernel_size= 4, stride= 2, padding= 1), nn.ReLU(), nn.Dropout(0.3), nn.Conv2d(32, 16, 4, 2, 1), nn.ReLU(), nn.Dropout(0.3), nn.Conv2d(16, 1, 1), nn.Sigmoid())\n",
    "        self.set_para()\n",
    "    def set_para(self):\n",
    "        nn.init.xavier_normal_(self.cv[0].weight, nn.init.calculate_gain('relu'))\n",
    "        nn.init.xavier_normal_(self.cv[3].weight, nn.init.calculate_gain('relu'))\n",
    "        nn.init.xavier_normal_(self.cv[6].weight, nn.init.calculate_gain('sigmoid'))        \n",
    "    def forward(self, x):\n",
    "        return self.cv(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1dcde077-4cd7-4640-bc42-e8e7233e27c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "seed= 1\n",
    "random.seed(seed)\n",
    "np.random.seed(seed)\n",
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed(seed)\n",
    "torch.cuda.manual_seed_all(seed)\n",
    "os.environ['PYTHONHASHSEED'] = str(seed)    \n",
    "torch.backends.cudnn.deterministic = True\n",
    "pic, label= load_mnist_data(train_dir)\n",
    "random_num= torch.rand(len(pic))\n",
    "train_flag, valid_flag, test_flag= random_num< train_ratio, (train_ratio<= random_num)& (random_num< train_ratio+ valid_ratio), train_ratio+ valid_ratio<= random_num\n",
    "train_set, valid_set, test_set= torch.utils.data.TensorDataset(pic[train_flag], label[train_flag]), torch.utils.data.TensorDataset(pic[valid_flag], label[valid_flag]), torch.utils.data.TensorDataset(pic[test_flag], label[test_flag])\n",
    "train_loader, valid_loader, test_loader= torch.utils.data.DataLoader(train_set, batch_size= batch_size, shuffle= True), torch.utils.data.DataLoader(valid_set, batch_size= batch_size, shuffle= False), torch.utils.data.DataLoader(test_set, batch_size= batch_size, shuffle= False)\n",
    "vqgan, discriminator= VQGAN().to(device), Discriminator().to(device)\n",
    "opt_vq, opt_disc= torch.optim.Adam(vqgan.parameters(), lr= lr), torch.optim.Adam(discriminator.parameters(), lr= lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8329ff7f-f0a2-4ac9-a91e-b05ef7a5771b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, step: 1, vq_loss: -2.8742661476135254, gan_loss:0.9700944423675537\n",
      "epoch: 1, early stopping time/full time: 0/80, valid loss: -1999.8682861048728\n",
      "epoch: 2, step: 1, vq_loss: -15999.9462890625, gan_loss:1.0\n",
      "epoch: 2, early stopping time/full time: 0/80, valid loss: -1999.8697509765625\n",
      "epoch: 3, step: 1, vq_loss: -15999.958984375, gan_loss:1.0\n",
      "epoch: 3, early stopping time/full time: 0/80, valid loss: -1999.8719329833984\n",
      "epoch: 4, step: 1, vq_loss: -15999.974609375, gan_loss:1.0\n",
      "epoch: 4, early stopping time/full time: 0/80, valid loss: -1999.87255859375\n",
      "epoch: 5, step: 1, vq_loss: -15999.9794921875, gan_loss:1.0\n",
      "epoch: 5, early stopping time/full time: 0/80, valid loss: -1999.872817993164\n",
      "epoch: 6, step: 1, vq_loss: -15999.982421875, gan_loss:1.0\n",
      "epoch: 6, early stopping time/full time: 0/80, valid loss: -1999.873062133789\n",
      "epoch: 7, step: 1, vq_loss: -15999.984375, gan_loss:1.0\n",
      "epoch: 7, early stopping time/full time: 0/80, valid loss: -1999.8731079101562\n",
      "epoch: 8, step: 1, vq_loss: -15999.984375, gan_loss:1.0\n",
      "epoch: 8, early stopping time/full time: 0/80, valid loss: -1999.8731231689453\n",
      "epoch: 9, step: 1, vq_loss: -15999.9853515625, gan_loss:1.0\n",
      "epoch: 9, early stopping time/full time: 0/80, valid loss: -1999.873275756836\n",
      "epoch: 10, step: 1, vq_loss: -15999.986328125, gan_loss:1.0\n",
      "epoch: 10, early stopping time/full time: 1/80, valid loss: -1999.8732452392578\n",
      "epoch: 11, step: 1, vq_loss: -15999.9853515625, gan_loss:1.0\n",
      "epoch: 11, early stopping time/full time: 2/80, valid loss: -1999.8732604980469\n",
      "epoch: 12, step: 1, vq_loss: -15999.986328125, gan_loss:1.0\n",
      "epoch: 12, early stopping time/full time: 0/80, valid loss: -1999.873291015625\n",
      "epoch: 13, step: 1, vq_loss: -15999.9873046875, gan_loss:1.0\n",
      "epoch: 13, early stopping time/full time: 0/80, valid loss: -1999.8733215332031\n",
      "epoch: 14, step: 1, vq_loss: -15999.986328125, gan_loss:1.0\n",
      "epoch: 14, early stopping time/full time: 0/80, valid loss: -1999.8733367919922\n",
      "epoch: 15, step: 1, vq_loss: -15999.9853515625, gan_loss:1.0\n",
      "epoch: 15, early stopping time/full time: 1/80, valid loss: -1999.8733367919922\n",
      "epoch: 16, step: 1, vq_loss: -15999.986328125, gan_loss:1.0\n",
      "epoch: 16, early stopping time/full time: 2/80, valid loss: -1999.8733367919922\n",
      "epoch: 17, step: 1, vq_loss: -15999.986328125, gan_loss:1.0\n",
      "epoch: 17, early stopping time/full time: 3/80, valid loss: -1999.8733215332031\n",
      "epoch: 18, step: 1, vq_loss: -15999.9873046875, gan_loss:1.0\n",
      "epoch: 18, early stopping time/full time: 4/80, valid loss: -1999.8733367919922\n",
      "epoch: 19, step: 1, vq_loss: -15999.9853515625, gan_loss:1.0\n",
      "epoch: 19, early stopping time/full time: 5/80, valid loss: -1999.8733367919922\n",
      "epoch: 20, step: 1, vq_loss: -15999.9873046875, gan_loss:1.0\n",
      "epoch: 20, early stopping time/full time: 6/80, valid loss: -1999.8733367919922\n",
      "epoch: 21, step: 1, vq_loss: -15999.986328125, gan_loss:1.0\n",
      "epoch: 21, early stopping time/full time: 7/80, valid loss: -1999.8733367919922\n",
      "epoch: 22, step: 1, vq_loss: -15999.9873046875, gan_loss:1.0\n",
      "epoch: 22, early stopping time/full time: 8/80, valid loss: -1999.8733367919922\n",
      "epoch: 23, step: 1, vq_loss: -15999.9873046875, gan_loss:1.0\n",
      "epoch: 23, early stopping time/full time: 9/80, valid loss: -1999.8733367919922\n",
      "epoch: 24, step: 1, vq_loss: -15999.9873046875, gan_loss:1.0\n",
      "epoch: 24, early stopping time/full time: 10/80, valid loss: -1999.8733367919922\n",
      "epoch: 25, step: 1, vq_loss: -15999.9873046875, gan_loss:1.0\n",
      "epoch: 25, early stopping time/full time: 11/80, valid loss: -1999.8733367919922\n",
      "epoch: 26, step: 1, vq_loss: -15999.9853515625, gan_loss:1.0\n",
      "epoch: 26, early stopping time/full time: 12/80, valid loss: -1999.8733367919922\n",
      "epoch: 27, step: 1, vq_loss: -15999.986328125, gan_loss:1.0\n",
      "epoch: 27, early stopping time/full time: 13/80, valid loss: -1999.8733367919922\n",
      "epoch: 28, step: 1, vq_loss: -15999.9873046875, gan_loss:1.0\n",
      "epoch: 28, early stopping time/full time: 14/80, valid loss: -1999.8733215332031\n",
      "epoch: 29, step: 1, vq_loss: -15999.9873046875, gan_loss:1.0\n",
      "epoch: 29, early stopping time/full time: 0/80, valid loss: -1999.8733673095703\n",
      "epoch: 30, step: 1, vq_loss: -15999.9873046875, gan_loss:1.0\n",
      "epoch: 30, early stopping time/full time: 1/80, valid loss: -1999.8733520507812\n",
      "epoch: 31, step: 1, vq_loss: -15999.9853515625, gan_loss:1.0\n",
      "epoch: 31, early stopping time/full time: 0/80, valid loss: -1999.8734588623047\n",
      "epoch: 32, step: 1, vq_loss: -15999.9873046875, gan_loss:1.0\n",
      "epoch: 32, early stopping time/full time: 1/80, valid loss: -1999.8734436035156\n",
      "epoch: 33, step: 1, vq_loss: -15999.9873046875, gan_loss:1.0\n",
      "epoch: 33, early stopping time/full time: 2/80, valid loss: -1999.8734436035156\n",
      "epoch: 34, step: 1, vq_loss: -15999.986328125, gan_loss:1.0\n",
      "epoch: 34, early stopping time/full time: 3/80, valid loss: -1999.8734588623047\n",
      "epoch: 35, step: 1, vq_loss: -15999.9873046875, gan_loss:1.0\n",
      "epoch: 35, early stopping time/full time: 0/80, valid loss: -1999.873550415039\n",
      "epoch: 36, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 36, early stopping time/full time: 1/80, valid loss: -1999.8735046386719\n",
      "epoch: 37, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 37, early stopping time/full time: 2/80, valid loss: -1999.8734893798828\n",
      "epoch: 38, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 38, early stopping time/full time: 3/80, valid loss: -1999.8735046386719\n",
      "epoch: 39, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 39, early stopping time/full time: 4/80, valid loss: -1999.8734893798828\n",
      "epoch: 40, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 40, early stopping time/full time: 5/80, valid loss: -1999.8734893798828\n",
      "epoch: 41, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 41, early stopping time/full time: 6/80, valid loss: -1999.8734893798828\n",
      "epoch: 42, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 42, early stopping time/full time: 7/80, valid loss: -1999.8734741210938\n",
      "epoch: 43, step: 1, vq_loss: -15999.9873046875, gan_loss:1.0\n",
      "epoch: 43, early stopping time/full time: 8/80, valid loss: -1999.8735046386719\n",
      "epoch: 44, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 44, early stopping time/full time: 9/80, valid loss: -1999.8734588623047\n",
      "epoch: 45, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 45, early stopping time/full time: 10/80, valid loss: -1999.87353515625\n",
      "epoch: 46, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 46, early stopping time/full time: 11/80, valid loss: -1999.8734741210938\n",
      "epoch: 47, step: 1, vq_loss: -15999.9873046875, gan_loss:1.0\n",
      "epoch: 47, early stopping time/full time: 12/80, valid loss: -1999.8735046386719\n",
      "epoch: 48, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 48, early stopping time/full time: 13/80, valid loss: -1999.8735046386719\n",
      "epoch: 49, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 49, early stopping time/full time: 14/80, valid loss: -1999.8735046386719\n",
      "epoch: 50, step: 1, vq_loss: -15999.9873046875, gan_loss:1.0\n",
      "epoch: 50, early stopping time/full time: 15/80, valid loss: -1999.8735046386719\n",
      "epoch: 51, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 51, early stopping time/full time: 16/80, valid loss: -1999.8735046386719\n",
      "epoch: 52, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 52, early stopping time/full time: 17/80, valid loss: -1999.8734588623047\n",
      "epoch: 53, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 53, early stopping time/full time: 18/80, valid loss: -1999.8735046386719\n",
      "epoch: 54, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 54, early stopping time/full time: 19/80, valid loss: -1999.8734588623047\n",
      "epoch: 55, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 55, early stopping time/full time: 20/80, valid loss: -1999.8734893798828\n",
      "epoch: 56, step: 1, vq_loss: -15999.9873046875, gan_loss:1.0\n",
      "epoch: 56, early stopping time/full time: 21/80, valid loss: -1999.8734893798828\n",
      "epoch: 57, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 57, early stopping time/full time: 22/80, valid loss: -1999.8734893798828\n",
      "epoch: 58, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 58, early stopping time/full time: 23/80, valid loss: -1999.8735046386719\n",
      "epoch: 59, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 59, early stopping time/full time: 24/80, valid loss: -1999.8735046386719\n",
      "epoch: 60, step: 1, vq_loss: -15999.9873046875, gan_loss:1.0\n",
      "epoch: 60, early stopping time/full time: 25/80, valid loss: -1999.8734741210938\n",
      "epoch: 61, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 61, early stopping time/full time: 26/80, valid loss: -1999.8734893798828\n",
      "epoch: 62, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 62, early stopping time/full time: 27/80, valid loss: -1999.8734741210938\n",
      "epoch: 63, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 63, early stopping time/full time: 28/80, valid loss: -1999.8734741210938\n",
      "epoch: 64, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 64, early stopping time/full time: 29/80, valid loss: -1999.8734893798828\n",
      "epoch: 65, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 65, early stopping time/full time: 30/80, valid loss: -1999.8734893798828\n",
      "epoch: 66, step: 1, vq_loss: -15999.9873046875, gan_loss:1.0\n",
      "epoch: 66, early stopping time/full time: 31/80, valid loss: -1999.8734741210938\n",
      "epoch: 67, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 67, early stopping time/full time: 32/80, valid loss: -1999.8734588623047\n",
      "epoch: 68, step: 1, vq_loss: -15999.9873046875, gan_loss:1.0\n",
      "epoch: 68, early stopping time/full time: 33/80, valid loss: -1999.8734588623047\n",
      "epoch: 69, step: 1, vq_loss: -15999.9873046875, gan_loss:1.0\n",
      "epoch: 69, early stopping time/full time: 34/80, valid loss: -1999.8734893798828\n",
      "epoch: 70, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 70, early stopping time/full time: 35/80, valid loss: -1999.8734893798828\n",
      "epoch: 71, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 71, early stopping time/full time: 0/80, valid loss: -1999.8735656738281\n",
      "epoch: 72, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 72, early stopping time/full time: 1/80, valid loss: -1999.8735046386719\n",
      "epoch: 73, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 73, early stopping time/full time: 2/80, valid loss: -1999.87353515625\n",
      "epoch: 74, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 74, early stopping time/full time: 3/80, valid loss: -1999.873550415039\n",
      "epoch: 75, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 75, early stopping time/full time: 0/80, valid loss: -1999.8735809326172\n",
      "epoch: 76, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 76, early stopping time/full time: 1/80, valid loss: -1999.8735809326172\n",
      "epoch: 77, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 77, early stopping time/full time: 0/80, valid loss: -1999.8736267089844\n",
      "epoch: 78, step: 1, vq_loss: -15999.990234375, gan_loss:1.0\n",
      "epoch: 78, early stopping time/full time: 1/80, valid loss: -1999.8736114501953\n",
      "epoch: 79, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 79, early stopping time/full time: 2/80, valid loss: -1999.8736114501953\n",
      "epoch: 80, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 80, early stopping time/full time: 3/80, valid loss: -1999.8736114501953\n",
      "epoch: 81, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 81, early stopping time/full time: 4/80, valid loss: -1999.8736114501953\n",
      "epoch: 82, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 82, early stopping time/full time: 5/80, valid loss: -1999.8736114501953\n",
      "epoch: 83, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 83, early stopping time/full time: 6/80, valid loss: -1999.8736114501953\n",
      "epoch: 84, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 84, early stopping time/full time: 7/80, valid loss: -1999.8736114501953\n",
      "epoch: 85, step: 1, vq_loss: -15999.990234375, gan_loss:1.0\n",
      "epoch: 85, early stopping time/full time: 8/80, valid loss: -1999.8736114501953\n",
      "epoch: 86, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 86, early stopping time/full time: 9/80, valid loss: -1999.8736114501953\n",
      "epoch: 87, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 87, early stopping time/full time: 10/80, valid loss: -1999.8736114501953\n",
      "epoch: 88, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 88, early stopping time/full time: 11/80, valid loss: -1999.8736114501953\n",
      "epoch: 89, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 89, early stopping time/full time: 12/80, valid loss: -1999.8735961914062\n",
      "epoch: 90, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 90, early stopping time/full time: 13/80, valid loss: -1999.8735961914062\n",
      "epoch: 91, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 91, early stopping time/full time: 14/80, valid loss: -1999.8736114501953\n",
      "epoch: 92, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 92, early stopping time/full time: 15/80, valid loss: -1999.8736114501953\n",
      "epoch: 93, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 93, early stopping time/full time: 16/80, valid loss: -1999.8736114501953\n",
      "epoch: 94, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 94, early stopping time/full time: 17/80, valid loss: -1999.8736114501953\n",
      "epoch: 95, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 95, early stopping time/full time: 18/80, valid loss: -1999.8736114501953\n",
      "epoch: 96, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 96, early stopping time/full time: 19/80, valid loss: -1999.8736114501953\n",
      "epoch: 97, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 97, early stopping time/full time: 20/80, valid loss: -1999.8735961914062\n",
      "epoch: 98, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 98, early stopping time/full time: 21/80, valid loss: -1999.8736114501953\n",
      "epoch: 99, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 99, early stopping time/full time: 22/80, valid loss: -1999.8736267089844\n",
      "epoch: 100, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 100, early stopping time/full time: 23/80, valid loss: -1999.8735961914062\n",
      "epoch: 101, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 101, early stopping time/full time: 24/80, valid loss: -1999.8735809326172\n",
      "epoch: 102, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 102, early stopping time/full time: 25/80, valid loss: -1999.8735961914062\n",
      "epoch: 103, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 103, early stopping time/full time: 26/80, valid loss: -1999.8735961914062\n",
      "epoch: 104, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 104, early stopping time/full time: 27/80, valid loss: -1999.8735961914062\n",
      "epoch: 105, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 105, early stopping time/full time: 28/80, valid loss: -1999.8736114501953\n",
      "epoch: 106, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 106, early stopping time/full time: 29/80, valid loss: -1999.8735961914062\n",
      "epoch: 107, step: 1, vq_loss: -15999.990234375, gan_loss:1.0\n",
      "epoch: 107, early stopping time/full time: 30/80, valid loss: -1999.8736114501953\n",
      "epoch: 108, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 108, early stopping time/full time: 31/80, valid loss: -1999.8735961914062\n",
      "epoch: 109, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 109, early stopping time/full time: 32/80, valid loss: -1999.8735809326172\n",
      "epoch: 110, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 110, early stopping time/full time: 33/80, valid loss: -1999.8735961914062\n",
      "epoch: 111, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 111, early stopping time/full time: 34/80, valid loss: -1999.8736114501953\n",
      "epoch: 112, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 112, early stopping time/full time: 35/80, valid loss: -1999.8736267089844\n",
      "epoch: 113, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 113, early stopping time/full time: 36/80, valid loss: -1999.8736114501953\n",
      "epoch: 114, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 114, early stopping time/full time: 37/80, valid loss: -1999.8735961914062\n",
      "epoch: 115, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 115, early stopping time/full time: 38/80, valid loss: -1999.8735961914062\n",
      "epoch: 116, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 116, early stopping time/full time: 39/80, valid loss: -1999.8735961914062\n",
      "epoch: 117, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 117, early stopping time/full time: 40/80, valid loss: -1999.8735961914062\n",
      "epoch: 118, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 118, early stopping time/full time: 41/80, valid loss: -1999.8735809326172\n",
      "epoch: 119, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 119, early stopping time/full time: 42/80, valid loss: -1999.8735961914062\n",
      "epoch: 120, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 120, early stopping time/full time: 43/80, valid loss: -1999.8736114501953\n",
      "epoch: 121, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 121, early stopping time/full time: 44/80, valid loss: -1999.8735961914062\n",
      "epoch: 122, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 122, early stopping time/full time: 45/80, valid loss: -1999.8736267089844\n",
      "epoch: 123, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 123, early stopping time/full time: 46/80, valid loss: -1999.8735961914062\n",
      "epoch: 124, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 124, early stopping time/full time: 47/80, valid loss: -1999.8735961914062\n",
      "epoch: 125, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 125, early stopping time/full time: 48/80, valid loss: -1999.8735961914062\n",
      "epoch: 126, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 126, early stopping time/full time: 49/80, valid loss: -1999.8735961914062\n",
      "epoch: 127, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 127, early stopping time/full time: 50/80, valid loss: -1999.8735961914062\n",
      "epoch: 128, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 128, early stopping time/full time: 51/80, valid loss: -1999.8735961914062\n",
      "epoch: 129, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 129, early stopping time/full time: 52/80, valid loss: -1999.8735961914062\n",
      "epoch: 130, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 130, early stopping time/full time: 53/80, valid loss: -1999.8736114501953\n",
      "epoch: 131, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 131, early stopping time/full time: 54/80, valid loss: -1999.8735961914062\n",
      "epoch: 132, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 132, early stopping time/full time: 55/80, valid loss: -1999.8735961914062\n",
      "epoch: 133, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 133, early stopping time/full time: 56/80, valid loss: -1999.8736267089844\n",
      "epoch: 134, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 134, early stopping time/full time: 57/80, valid loss: -1999.8735961914062\n",
      "epoch: 135, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 135, early stopping time/full time: 58/80, valid loss: -1999.8735961914062\n",
      "epoch: 136, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 136, early stopping time/full time: 59/80, valid loss: -1999.8735961914062\n",
      "epoch: 137, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 137, early stopping time/full time: 60/80, valid loss: -1999.8736267089844\n",
      "epoch: 138, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 138, early stopping time/full time: 61/80, valid loss: -1999.8735961914062\n",
      "epoch: 139, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 139, early stopping time/full time: 62/80, valid loss: -1999.8735961914062\n",
      "epoch: 140, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 140, early stopping time/full time: 63/80, valid loss: -1999.8735961914062\n",
      "epoch: 141, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 141, early stopping time/full time: 64/80, valid loss: -1999.8735961914062\n",
      "epoch: 142, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 142, early stopping time/full time: 65/80, valid loss: -1999.8735961914062\n",
      "epoch: 143, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 143, early stopping time/full time: 66/80, valid loss: -1999.8736114501953\n",
      "epoch: 144, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 144, early stopping time/full time: 67/80, valid loss: -1999.8735961914062\n",
      "epoch: 145, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 145, early stopping time/full time: 68/80, valid loss: -1999.8735961914062\n",
      "epoch: 146, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 146, early stopping time/full time: 69/80, valid loss: -1999.8735961914062\n",
      "epoch: 147, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 147, early stopping time/full time: 70/80, valid loss: -1999.8735961914062\n",
      "epoch: 148, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 148, early stopping time/full time: 71/80, valid loss: -1999.8735961914062\n",
      "epoch: 149, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 149, early stopping time/full time: 72/80, valid loss: -1999.8735961914062\n",
      "epoch: 150, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 150, early stopping time/full time: 73/80, valid loss: -1999.8735961914062\n",
      "epoch: 151, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 151, early stopping time/full time: 74/80, valid loss: -1999.8736114501953\n",
      "epoch: 152, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 152, early stopping time/full time: 75/80, valid loss: -1999.8735961914062\n",
      "epoch: 153, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 153, early stopping time/full time: 76/80, valid loss: -1999.8735961914062\n",
      "epoch: 154, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 154, early stopping time/full time: 77/80, valid loss: -1999.8735961914062\n",
      "epoch: 155, step: 1, vq_loss: -15999.9892578125, gan_loss:1.0\n",
      "epoch: 155, early stopping time/full time: 78/80, valid loss: -1999.8735961914062\n",
      "epoch: 156, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 156, early stopping time/full time: 79/80, valid loss: -1999.8735961914062\n",
      "epoch: 157, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n",
      "epoch: 157, early stopping time/full time: 80/80, valid loss: -1999.8735961914062\n",
      "epoch: 158, step: 1, vq_loss: -15999.98828125, gan_loss:1.0\n"
     ]
    }
   ],
   "source": [
    "best_score, score, time, first= 0, 0, 0, True\n",
    "for epoch in range(epoch_num):\n",
    "    for step, (pic, label) in enumerate(train_loader):\n",
    "        pic= pic.view(pic.shape[0], 1, 28, 28).to(device)\n",
    "        decoded_pic, _, q_loss= vqgan(pic)\n",
    "        disc_real= discriminator(pic)\n",
    "        disc_fake= discriminator(decoded_pic)\n",
    "        # disc_factor, when to start the discriminator.\n",
    "        d_loss_real= torch.mean(F.relu(1.0- disc_real))\n",
    "        d_loss_fake= torch.mean(F.relu(1.0+ disc_fake))\n",
    "        # why does discriminator start later?\n",
    "        disc_factor= vqgan.adopt_weight(1.0, epoch* len(train_loader)+ step, threshold= 0)\n",
    "        gan_loss= disc_factor* 0.5* (d_loss_real+ d_loss_fake)\n",
    "        # the loss of generator,\n",
    "        # For Generator, care pseudo example, the harder it is to discriminate, the better.\n",
    "        recon_loss= vqgan.mse_loss(pic, decoded_pic)\n",
    "        λ= vqgan.calculate_lambda(recon_loss, -d_loss_fake)\n",
    "        vq_loss= recon_loss+ q_loss+ 1.0* λ* -d_loss_fake\n",
    "        # update discriminator\n",
    "        opt_disc.zero_grad()\n",
    "        gan_loss.backward(retain_graph= True)\n",
    "        # update generator\n",
    "        opt_vq.zero_grad()\n",
    "        vq_loss.backward()\n",
    "        opt_disc.step()\n",
    "        opt_vq.step()\n",
    "        if step% 100== 0:print(f'epoch: {epoch+ 1}, step: {step+ 1}, vq_loss: {vq_loss}, gan_loss:{gan_loss}')\n",
    "    vqgan.eval(); discriminator.eval(); val_loss= 0\n",
    "    for step, (pic, label) in enumerate(valid_loader):\n",
    "        pic= pic.view(pic.shape[0], 1, 28, 28).to(device)\n",
    "        decoded_pic, _, q_loss= vqgan(pic)\n",
    "        disc_real= discriminator(pic)\n",
    "        disc_fake= discriminator(decoded_pic)\n",
    "        gan_loss= 0.5* (d_loss_real+ d_loss_fake)\n",
    "        recon_loss= vqgan.mse_loss(pic, decoded_pic)\n",
    "        d_loss_real= torch.mean(F.relu(1.0- disc_real))\n",
    "        d_loss_fake= torch.mean(F.relu(1.0+ disc_fake))\n",
    "        λ= vqgan.calculate_lambda(recon_loss, -d_loss_fake)\n",
    "        vq_loss= recon_loss+ q_loss+ 1.0* λ* -d_loss_fake\n",
    "        val_loss+= (vq_loss.item()+ gan_loss.item())\n",
    "    val_loss= val_loss/ len(valid_loader)\n",
    "    # early stopping\n",
    "    if first==True or val_loss< best_score- 1e-4:\n",
    "        first= False\n",
    "        best_score= val_loss\n",
    "        time= 0\n",
    "        torch.save(vqgan, f'{pt_folder}//{pt_file4vqgan}')\n",
    "        torch.save(discriminator, f'{pt_folder}//{pt_file4disc}')\n",
    "    elif time>= patience:\n",
    "        vqgan= torch.load(f'{pt_folder}//{pt_file4vqgan}')\n",
    "        discriminator= torch.load(f'{pt_folder}//{pt_file4disc}')\n",
    "        break\n",
    "    else:\n",
    "        time+= 1\n",
    "    print(f'epoch: {epoch+ 1}, early stopping time/full time: {time}/{patience}, valid loss: {val_loss* 1.0/ len(valid_loader)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "990faa50-1860-4b54-9a37-67680a50dc2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "vqgan= torch.load(f'{pt_folder}//{pt_file4vqgan}')\n",
    "discriminator= torch.load(f'{pt_folder}//{pt_file4disc}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "56d55f0a-48c9-4221-bd6e-8fd01d4d29b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "vqgan.eval(); discriminator.eval(); pred= []\n",
    "for step, (pic, label) in enumerate(test_loader):\n",
    "    pic= pic.view(pic.shape[0], 1, 28, 28).to(device)\n",
    "    decoded_pic, _, q_loss= vqgan(pic)\n",
    "    disc_real= discriminator(pic)\n",
    "    disc_fake= discriminator(decoded_pic)\n",
    "    pred.append(decoded_pic.detach().cpu())\n",
    "pred= torch.cat(pred, dim= 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "21dddf1b-90d5-4faa-86ce-8cb23c683e88",
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_mnist(images):\n",
    "    _, figs = plt.subplots(1, len(images), figsize=(12, 12))\n",
    "    for f, img in zip(figs, images):\n",
    "        f.imshow(img.view(28, 28), cmap= 'gray')\n",
    "        f.axes.get_xaxis().set_visible(False)\n",
    "        f.axes.get_yaxis().set_visible(False)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "1a34a4f9-ab24-419f-9bea-8abea0a5371e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7oAAABnCAYAAAAuTD4BAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABy5ElEQVR4nO1992+j2ZXlYU4fc6ayqlSq6HK729neGax3gN0FBvvD/qW7iwUWmMFg7XHbbruDu7q6knIiJcaPOYrcH7Tn6pGt6q4sSnwHIKRSkRT1vvfdd8O551pGo9EIGhoaGhoaGhoaGhoaGhrXBNbL/gAaGhoaGhoaGhoaGhoaGu8SOtDV0NDQ0NDQ0NDQ0NDQuFbQga6GhoaGhoaGhoaGhobGtYIOdDU0NDQ0NDQ0NDQ0NDSuFXSgq6GhoaGhoaGhoaGhoXGtoANdDQ0NDQ0NDQ0NDQ0NjWsFHehqaGhoaGhoaGhoaGhoXCvY3/SFw+EQ2WwWfr8fFovlXX6ma4/RaIR6vY5MJgOr9fVyDXrd3xxvs+6AXvu3gV77y4Fe98uDXvvLgz5jLwd6z18e9J6/HOg9f3l45bUfvSEODg5GAPTjLR4HBwd63a/Iuuu112t/lR963fXaz+JDn7FXZ9312l/e2ut1v5x112v/Ydb+janLfr//TV+q8f/xJmuo1/3t8aZrqNf+7aHX/nKg1/3yoNf+8qDP2MuB3vOXB73nLwd6z18efmgN35i6rEvsb483WUO97m+PN11DvfZvD732lwO97pcHvfaXB33GXg6u+p63WCzyIEajkTymGXrPXw6u+p6/yvihNXzjQFdDQ0NDQ0NDQ0PjOsFiscBms30n0B0Ohzg9Pb3ET6ahofG60IGuhoaGhoaGhoaGBs4CXbvdLoGuxWLBaDTCYDDAcDic+qquhobGOXSgq6GhoaExtaCTqaExK1ADLABXgjJ7HWC1WmGxWBCPx7G6ugrDMOBwOOB0OjEajbC9vY2dnR0MBgP0+30MBoPL/sgaGho/AB3oamhoaGhMJdQ+Oe3oa8wC1P5Qde9ryuz7BenKVqsVqVQKv/nNb5BMJmEYBoLBIPr9Pv71X/8V5XIZ7XYbrVYLp6en2i5paEw5dKCroaGhoTEVIGXQZrPJz9TeuOFweImfTkPj/YL732q1wmq1Sp8oK4ikzerg6t3DarXC6XTCZrPB5/MhFAohEonAMAwEAgH0ej14vV4JhidpzRoaGtMJHehqaGhoaFwaLBaLOPYejweLi4uIx+M4PT1Fs9lEv99HrVZDsVhEp9PR1S2NawmLxQK3241EIgGPxwOv14tgMAiHw4FisYhcLodut4tOp4NutytJHx1kvRv4fD4sLi4iFArh7t27uHXrFpLJ5Jjashrganw4XMRymATvAzURpO8NDUAHuhoaGhoalwQ6LjabDXa7HYZhYH19HWtra+h0OiiVSmi1Wjg5OUGz2cRgMJDKrnZiNK4L6Ly73W6kUilEIhGEQiHMz8/D7XZjc3MTvV4PrVYL1WoV/X4fgO7dfRfg2nu9XqyuriKTyWB9fR03b95EPB5HvV5HtVrFYDAQpomu5n5YMBmqfiUmg9rJRKi+PhpTE+iqmRq73S70HX5PxbvT01Ocnp6i1+vprL6GhsalgAeu3X5mQofD4RitUB+u3w9SNElT9vv98Hq9CIVCSCQSiMfjaLVaGI1GcLvdaLVacLlc6HQ6AKBFYDSuHSwWCxwOB/x+P0KhEEKhEKLRKNxuN0zTRDKZRLPZhNVqFWe+3+9Ln6hK7df259VBW+50OhEIBBAOh0WEymKxoN/vo1qtol6vS18u11yv87uFOtbJ6XTC4/HAarXC4XDA4XAAwNjYJ3X92eIyHA5FHZsMCLa+6HNjHGqVfHItr9PevvRAl4tst9vhcDhgs9kQiUQQi8UkuxmLxdDv93FycgLTNNFoNLC3t4dqtXrtLshVgppV0+qQGrMA9SD2+/0Ih8Ow2WxoNptoNpsYDofodDro9XqX/VGnFnQsw+Ew4vE4fD4fHjx4gPX1dfh8PqysrCAej6Pdbksl99mzZ8jn87BYLGOVXW1rpg8XVVs0vh+k7gcCAdy7dw83btxAOBzGwsIC3G43VldXcefOHbRaLWxtbWF3dxedTgflclkqvLVabYzar9f+h2G1WuF2u+FwOBCPx3H//n3cvn0boVAIDocD/X4fu7u7+NOf/gTTNPH06VNUq1X0ej0MBgO9xu8ItBkul0uSDKurq7h37x68Xi/C4TBCoRCsVisAyFdVs4G+Z7vdlntiZ2cHGxsbaLVaqFQqqFarEgzPMlQNAPalq8l6agJcF3/+UgNdNZvABXc4HIjFYlhZWYHP58OtW7ewvLyMTqeDjY0NHB8fo1gsolAooFarAdCH6WVhcgTCJPR10bhuUPtJDcNAPB6H3W5HpVKRA2IwGOhA9yWgvbdarfD7/Uin0wiHw/jFL36BX/3qV/B4PAiFQvD5fGi1WojH42g0Guj1eohEIhLk6j656YN6HrBCoM+AH4ZqU3w+H5aXl3H37l2Ew2GhLs/NzWFlZQXtdhvhcBhutxvNZhMHBwew2+2SXKNglWa7vRpYRXe73QgEAlhZWcGdO3eErTMYDHBycoJHjx6hWCwim82i2Wzq9X0P4LUwDANutxs3btzAb37zG4TDYaRSKaRSKWFQTVZ0VbpyrVZDoVBAu93G559/jlarhVqthn6/j3q9jtFoNNOUc9Xe2O12SfSw4s0kgFr9vuprdamBrtPphN/vh8PhgM/nQzQahcvlQiaTwcLCAjwej/Sr9Ho9NJtNuN1u+Hw+lMtleDweNJtNlMtlcSyv+gWZdjCT5vF4EAgEZMac2+0GAFQqFVQqFblp9IGgcd1AR97v92NxcRFOpxNerxcOh0NaKtrttrZFF8BmswkNLRqNYmlpCeFwGNFoFF6vV7LLAIR61m630el0ZG6ltikfHmpbkdPpFOEwt9v9nSoLn9vr9VCv1+W6XacKwfuEqjBOp9TlconibzQaRSaTQavVwnA4hMPhQLPZRLfbFToz6Zsa3w+bzYZAIIBgMIhoNAqPxyN2nAm2SqWCer2OZrOJXq+n9+87gDrKyeVyIRQKweVyIRAIIB6PiyhhOBxGMBiEz+cTu6OCFUg1ucbz2GazIRaLYWFhAbVaDYPBAN1uF/1+H81mU9pgZgVsE3K5XEgkEggEArDb7fB6vbDb7ULJHw6HogcwGo1E/I5+DYXw1KB42u+JSwl0uVnD4TDu3buHUCiExcVFPHjwQGaWkTri9/thGAYGgwFu376NbreLUqmEW7duoVAoYGNjA//+7/+OYrGoOfjvGeyVsFqtmJ+fx09+8hNEo1EkEgnMzc1hNBrhD3/4A/74xz+i3W6jVqvJzaKhcR2gtlqsra3hn//5nxEMBrG9vY2dnR1Uq1V8/vnnME1TCyZdADo1Ho8HH3/8Mf75n/8ZoVBIWlQoNHJ6eopWq4V8Pi+Ks3Q42XOl1/bDQK0AqH2jKysrWF5elpYjh8MxFsjm83k8fvwY5XIZjUYDpVJJj8j5Hqj7vl6vw+v1AjhzUH0+H9xuN05PT+HxeLC0tIROp4OjoyNhuH322WfY3d1Fo9FAPp/X9OVXgMfjwa1bt7C6uoqFhQUkEgl4vV5Uq1U8fvwYpmni8ePH2N3dRbVaHVO71nh9MAlms9ng9XrhcrmQTqfx61//GvPz80LXp15DLBaTxKjT6QQw3ovLgIv2yWq1StJ5OBzC5XJhfn4ezWYTX331FR49eoR6vY6trS0cHx/PzP1BBpphGIjFYvgv/+W/4OHDh7Db7XC5XLDb7WNr2ul00G630ev1kMvlkM1m0Wq1sL29jaOjo7FWCZXyPK24tIquxWKB1+tFOp1GPB7H3bt38dOf/lScIGZk+CBGoxHK5TIAoFgsYjAY4PPPP4fVapUM6DQv+FUH6Q7BYBA3b95EKpXC4uIibt26heFwiGw2i6+//hoA0Gq1LvnTami8ezDYjUajuHfvHsLhsDj55XIZz549E8EYYPqznR8SdrsdHo8HhmFgYWEB9+/fRzAYhMvlgtPpxGg0EoEdjheqVquSgWdGXq/ph4Ma6LrdboRCIRiGgeXlZdy/fx9Op1OcJeC8yrK/v49CoSD/rlarUo0/PT3VZ/UF4P7vdrtC0Wcvndvtxmg0gsfjQSwWQ6/Xg2EYiEajCAQC2N7eRrFYxGg0GqN46jV+Odgqt7i4iFQqJZXDfr+PfD6PfD6Pk5MTSdZovBtQ/Mvj8SASieDOnTu4desWYrEYlpeX4fP5xgRq1SBscowQEw9MkjocDrhcLgBnidVoNCpq5eVyGZVKBblc7nL+8EsCxb28Xi9isRh+9KMf4R/+4R+kwqv2PLPPmQyGzc1NBAIBEWNrNBpot9sSCPN104wPGuiqc+Lo6Ny8eROJRAKpVAqGYYwdmKrS8iT8fj+GwyEymYyIBzQaDZimicFgIBdCZ47fPXiQ+nw++P1+yTaPRiPpp/B4POh0OtITofH2uEj8Sz0MLtrrOth6t5jMJgPn2dJEIiF9dqoyqsZ3kwORSET6D6mwD5wFQDxQT05OsLu7i6OjI8koqxVBjfcHVXQtEAggmUzC4/EgnU5jaWkJXq8XKysrSKfTY0KSwLmzNBwOcf/+faRSKRwfH8PhcEi/HBkPgLZNqsPebDaxt7cnlEEGtB6PR+wKwapYMBjEYDDA2toanE6nBLuVSgXtdhv1el3boQmwgOLxeJBIJGR+LgA0m02YponDw0PkcjlhC2q8HmjznU6ntKQEAgHx82OxGPx+P+bm5jA3N4doNArDMGCxWIRm3G63cXp6imq1KomybreLXq8nrS2s6JIKzVY6qvkHg0FYrVbEYjHcuHEDpVIJh4eHKJVKGAwG6HQ6M8EEZfWbtOR6vS596JxsQ5vMhDL/j4kHtmq1Wi0cHR3BNE20221UKhURwptGe/7BAl1u+lAohF/84hdYWVnB3NwcPv74Y8RiMREh4YIDZxeGi62+x2g0QiKRkJ5em80G0zRxcHCAzc1NuQjZbFbPXHzHYNXc5XIhHo8jk8kgHo8jHA5jNBpheXkZDx8+lAzoycmJziq/A6iOJ/8NYIzxMDniBvjuTDkV+pq8Pph8U6+D3W4Xyls+n0c0GpXRFDrYHa8I3rx5E//9v/93pNNpLCwsSJ8/cLa23W4XuVwO+Xweu7u7+MMf/oDt7W3U63WUy+WpPkyvE9g7Z7fbsbKygn/4h39AIpHAwsIC1tbW4Ha74fF44PF4xkQlgfPAbXl5Gevr6+h2u3jx4gX+9Kc/oVwuY2trC0+ePEG325XnzzrIYCgWi/j000/x9ddf4/79+/B6vXLGzs3NCX2TiQVWc3kG1+t17O3tIRAIIJvNIpfLYXNzU2sGKGDw5XK5EA6HcffuXfzqV7+SanqlUsHe3h7+9re/YX9/X3p1NV4dKpU4GAwiHA6LuOzq6ip8Ph+WlpYQi8Wk6OX3+wGc2YNer4d8Po+DgwM0m008f/4cz58/R7fbRaVSQaPRkF5Szje22+2SmGOf78cff4xf//rXcLvdWFtbw+LiIvL5PMrlsvRdFwqFax/ock1ZkS2VSpJ8JNWbzwMwprHDpMRwOMTCwgIAoFqt4smTJ8jlcigUCtKiwtdNm6354IGu0+lEPB7HwsICMpkMMpkMotHo2EZVm6KZ2QHOqQmsDFssFkQiESwsLCAUCuH09BSVSgVOpxOlUgk2m20saAb0ofquoFJP3G630A4Nw0A4HMZwOBSBKo2X42XqsZM/Z8ZSdSgZ/E7eN+os18n9T0zOTNN4dUyuLe2RzWZDt9sVgR6tDHwOBruBQABLS0uYn59HMBiUnv/Jqla1WkWlUkE+n0c2m0Wv1xNBDI33D3Xkn9/vx/z8PDKZDJaWlnDz5k2x7S+zI8PhUObBsvKyvb0Nm82Gk5MTqb7o63kGrl2n00E+n0elUkE0GkW5XIbX64XX60W/3x87B5iMIFXTbrdLD2kikUCn00Gj0ZDzQf09sw6em06nU2Z3dzodFItFdLtdNBoNFItF6XXW+/T1QN+E1FjDMOD3+5FKpbC0tAS/34/V1VUkk0k4nU4Eg0GhjLfbbRGMYkC6v7+PjY0NtNttFItFGRPEYEyt6IbDYVSrVXi9XiwtLUkQGwgExp5jGAYASCvkdQdF7nq9HtrtNlqtliTkuUaTBRIKfTGx5vf74fF4YJrmmOgsK/bTal8+SKCrbnr2l6TTaanI8sBjcFutVmGapmR1SqUSgPNB0V6vF8lkEj6fD71eT8ZRsBG91WrJodxut3F0dIRCoSC0h+uevXlfUKkN7XYb5XJZaCikbbLayxtDdWI1zh1+tZeEo7V4LzB5wLW02+3S20jlPNI9+WCgO7m3adhI+eT+V2fNNZtNNBoNuf/0of5yqImDarWK/f19dDodBAIBBAIBqXJx0L12ks4EX5hhT6fTCAQC0gsHnK1lq9UShdMXL17g2bNnOD4+RqlUGtu/Gu8XDIgMw8Dq6iqCwSDu3r2L5eVlJJNJRCIRaS3q9/tyXfg9A2R1fAWrOqurq4hEIjBNE1tbW9LrNWvqp98HruVwOMTx8TH+9re/YXt7G7FYDHNzc8KkisVicDqdci+NRiMJKqLRKO7evYtEIoFIJILBYCCJI9I1Zxk2mw2RSEQYCky4tVotFAoFlMtl5PN5ocdqu/NqoFipzWaTYhZ99VQqBa/Xi9XVVWlZCYVCcgZUq1UAZwJ2GxsbqNfryOfzODw8RLvdxt7eHgqFgkxf4T0yyWLjnPVisQi3243NzU387W9/QzAYlASr2+3G+vo6RqMRTk5OhAo9rdXIdwEy0TqdDqrVKh49eoRmsym9/5Nq1gRtOMcQsX/a4/Fgbm5OknD5fB5utxv1eh2FQmHqdDQ+WKBL1bRAIID5+Xmsra3JIjETyR6svb09bG1toV6v48mTJ9ja2sJwOJTAKZlM4mc/+xnS6TRCoRAymQzcbjeWlpbw4x//GP1+X5TCTNPE73//e3z11Vfo9XowTVOrEb4h1CxPo9HA0dGRVG7n5+clO+R2u4UOodLa9JpjzBFk9ZsZTRp+Oigcv+VyuUS0geMPKNbG9wRwYT87K2S9Xg+dTgemaaLb7eL4+BhbW1toNps4OjrCwcEBer2eHO76Wr0cTPgUCgV8++23yOfzuHv3LjKZDHq9nijFs8I76wgEAlhfX0ckEsHNmzeFbsn9e3p6KvoKuVwOf/nLX/DXv/4VzWYTx8fHMkZF78n3D7XF6Cc/+YkoKz948ECCXNLcOIJlMBigXq9L9ZBjojguhCyuH//4x2i32zBNE8+ePYPL5UK5XEa329XX9v+DiqcWiwXb29vI5/NSVadQ58OHD/GjH/0IhmHg1q1bWFhYEIaVxWJBJpOB3+9Hr9fD06dPheH27Nkz1Gq1mfd/7HY7MpkM7ty5g0wmg1gsBpfLhcFggIODA+zv7+Pg4ED8UY1XA3s5nU4nbt++jd/97neIRqNIp9OYn58Xf8br9Y4xDNrtNgqFAlqtFh49eoT/9b/+F3K5HBqNBqrVqvSV0k78kMrvYDBAo9GQCqNpmgiHw/jd736HdDoNwzDw85//HA8ePMCLFy9E0Z/U3uuYmCZ1mUJ3//Zv/4ZPP/10rPCisgVZIbfb7ZKoCAaDEr/5/X6sr69jOBwimUyi0WggFovh4OAA9Xr9OxXiy8YHoy6rC0fHXZ2LxUxmt9sVMQAOfs5mswDOK7rAWQaINw0HfvNCqXPoOM7CMAy0Wq2x4GBaLsJVAwNeVYJcnWfG6r16A806uBbMjKliXg6HA4FAAOFwWPoh1Iy91+uF2+2WXnZ1TpwKNdDlug8GA5mx2G634Xa70el0cHp6inK5DIfDgWq1KvcP70NgeozUNIIZ0nq9DrfbPUYrpJ3jPTDr4F4n88bhcAidko4L7T4D3lKpNCY6olkhHwZ0fKisH41G5fzkuBvaCTqg/X4frVZLzlf+P78CkGBNtWmdTmesl13jDNzrnU4HvV4PFosFjUYDzWYTbrcbqVRKRjXV63W0221hBdH2BAIBDIdDRKNRRCIRjEYj+Hw+sUmznDiyWCzweDwIBoMwDEP80OFwiHa7jUajIWfkrK7Rm4AVXQa0ZB4kk0kkk0lhsDmdTrH5ZII0Gg3RYcjlcjg6OpJr8bp7dTAYiEBVvV4XMbFGo4F+vz82trRUKsHn84lKM6/7dQRtMWdtA+Oipvyq+vBMbNLfVP0cl8sFi8Uia0lR2mn0eT5oRdftdsPlcgm1lcJTVFXb399Hs9nE06dPpbR+eHgoyr1cQLfbLdVEAFhYWBBaJ8d8RCIR2Gw2hMNh/PznP0ckEkGxWMRf/vIX7O7uykGtDdnrgevV7XZRq9XgdrvRbDbFODCL3+v1JKADMJMHK40Hg1dKu6+vr8sQdIrxeL1e+Hw+2O12MRqklTidTvk5D2VWhdW+XOD8+tChYYDt9XpljtpgMEA4HEYkEkGn08GNGzewt7eHZrOJZ8+eYXt7Ww4LTdt6Oag2aLVa5R6gIrnL5ZLeoVnFJA12YWFB6Jek2vf7fXQ6HWxtbeHFixfI5/M4OjqSrLDegx8OTMQxUMpkMlheXpakG3BOV+73+zg8PMT+/j663S7K5TJqtRocDodUHsPhMICziv7p6akEY8vLy/jtb38L0zTx+eefo1arSZJOX+tzMAlksVjQ6/VkhvSzZ8/QaDTg9Xqxvb0tUw5SqZQ4naSKhsNh3L9/H7VaDe12WyoujUZjZmfc22w2hEIhLCwsSPKYwkalUgm5XE4SCRqvBvoYpIIvLy+LknIwGBzzVzqdDjqdDvb391EqlVAul/Htt9+iWCzi8PAQ+Xxe6Mlvk+Ak8zCXy6HZbOLFixdIJpMwDENaJyORCG7fvg0AODk5wdOnT2fiul80nYMJR37lc9Qz2OFwSFsWn6O23tHfnzZ8kE+lChdxSDSddwa6pmlie3tbBnR/9dVXIo3fbDYl0GW24fDwEJ1OBy6XS+TBGejywA6Hw+j1evB6vbh9+zZ2dnZweHg4psZ8XbM37xNUJ6xWq7Db7WNO/mSgS/rILPYFMTvGfoZYLIZbt27hP//n/yx0ewa0qnryZHaN/1arYHyQBaHOkGZVhvcClVFV9Ho9rK2tYTAY4OTkBEdHRyLfn81m5T2143kxSLkqFosYDodC4RwOh1K1Z1Z5lmGxWGAYBm7cuIG1tTWkUinpRafDU6/XsbGxgT/96U+oVCrY399HtVoVR38WnfHLgKobQAGqGzduwO12j4mWcKbx3t4evvnmG7TbbeTzeZimKUk9n88nFFr1fZ1OJ1ZWVmAYBiqVCsrlMl68eCFq2trejIN2nuwGAKjVanjx4oXMgA2FQggEArh//z7S6TQymQx8Ph+8Xq+MbKFa7dOnT1Eul8V+zeK9ZbPZZF5rKBSC1+sVynixWEQ2m0WlUplJn+VNQN/C5/NheXkZ8XgcN27cEJFYh8Mh7Q5MzNdqNTx//hybm5vI5XL485//jKOjI6EPvwvq62g0EsaDaZp48eIF/H6/BN/z8/My7i4YDOLFixfY399HrVZ7+0W5IniZmCDZOCpVHICwCZl8o4+jCuLxvabJtnxw6vKkgA4Xkk3mtVoNzWZzbCCxGoyyEtDtdoU6y0PAZrPJwvNByk6/3xfKlMfjkX5EjTeDSl1WG8/V5vVZpi8zIUNlTNIAI5GIVFM5740BLIMi1eGbzKzxKymd3McUnSLVTaWHMpvK73ktKBzh8/kQDAYxGo2kFUClt2hcDLJCOAJNDcomkxSzCP79DPzJTlCDfwqAqElNdVbuNB2W1x2qijsdFwanqm3inqeqb7vdRrPZlGtHNoPP50OtVpNWJXXOpWEYOD09FTElsiJ0cHEx1HuBdPFeryfTDjhxwuVyiSBnq9USJ5TzvVldr1arM2mbVJvkdDolAFMVaUmp1bbn9cF9qvbVUkkZOGeEsD2lXC5LmyLnPb9LpqV6RnOGt8PhEN/farVKwYEJWI0z0CelvWb/tdpuyuvFaz2tLNkPqrpM55sPAFJZLZVKePHiBQqFgtCVadAnez8BoF6vAwCOj4+xubmJcrmMUCgkMyzZU8SKgt1uR7vdxs2bN9FoNFCpVLC9vT1T2Zt3AV6LVquFbDaLer2OW7duodvtirCYYRjSD6pS3mYJNptN6Mlzc3P4zW9+g/X1dVEc9/v9ck+oFVy1inV6eipqtMzIU1Dq8PBQMp8MelkhA87vK7VvlAG3y+VCMpnEjRs35JqlUin4fD4kEgmEw2EZ1K6TQS8HFaup2NntdseM/TQa/A8FOpNU2g8EAlI94T6nCmSr1UKlUsHx8bEETrO8dpcFjo+gXgCrgkyUkUHSarVE2fTo6AitVgumaaLRaMBiscA0TdjtdhwfH6NSqSAQCGBxcVEElKxWKyKRCNxuN+7cuYNKpQLTNPH06VMcHBxc9jJcCaijuAaDgQi2bW9vizhVKpVCMpnE6uoqPB4PVlZW8Nvf/hblchmffvopCoUCgNlpK+JZyDY6BjcUL6rVaqJMzSkEb4PJUYCEejZch3Xn39BoNLC1tSVKxvQF1YIIg9pWq4WDgwOUy2U0Gg2Uy+XvJIvf1Wej3To5OcGzZ8+QTCbxox/9SPwmt9stY3N0oHsGJjo9Hg/u3r2Lf/zHf0QwGMTCwgJsNht6vZ4UJY+OjvDs2TPs7u5OrbjgB6voqpliBroUvhkMBjBNE7u7uzg+PkaxWBzL7qp0TFJh2V9SKBSwu7sL0zSRSCRwenoqDdEUz+C4lna7jaWlJTQaDWSzWWSzWR3ovgFIezo5OZGxBf1+H6enpzIGhxR1h8MhAdesiI4wKWMYBqLRKObn5/Hxxx/j448/liQMK6uq+uAkjYSBbqPRQKPRwMHBAarVKo6OjvD111/DNM2xcRQc1XF6eirfs8LOz5NOp+Hz+XDv3j2EQiGMRuezj6nsHAqFYLfbJZmkcTHoYDqdThGO4X2gKkPOyr5XofZ7co5iMBgU0TMGuhRJq1arKJfLkjAArocTeJXAM5OjstQqB+0UZzA2m01JTlA0hskx2jKPx4NcLge3240f//jHmJubg8ViQSAQQDAYhNfrxY0bN2Ssy/HxMQ4PD/V1f0WoNt9isaBSqcBms6FUKsEwDBQKBQwGA6ysrMhkBOBsbuju7i6+/vrrmWJOqIKoZFM5nU6x42QUVqtV2ctvCpXRM8numWwJui5r32q1cHh4KGtK7REmD7rdLrLZrOzLTqcjhaz3JfzFhNBgMECxWJQzp1qtotfrSbvdNAspfWioLCy2Hf3qV7+CYRiSDGCSjbPut7e3sbW1JT7QtOGDBboMVCeprN+3uVUHka/h62m0AEifolrRovPPjctMHp36er0ulJ5ZMfTvEnRUVerm6emp0GapvudyucaExK47VNU6wzAQi8UQiURk5IZKHVYpyqSinZ6eCnWfVVxSOo+OjtBoNFAoFGCa5phgD3CmGNjtdoVW3u12x4SrqELIAJpBGa+Pek9p/DBI3VED28nZfsD1cWTeBBfZbeA8kcP9y4TA5LiD17XNF1VQJqsp/Jn6WSafO9kfPAvXkDRXlWI+maBkdUSl6/Ma8t+8H8j8YV9vs9mU1iGuP6sp7XZbxuPMwlq/K6hrRVtOWjnZQMDZvmZ/XbfblfntahLjukP1QVl0sdls0gpHJ52+zOvaHQYIFEZlkk9tCQAwlhClwNh16E+nLQDOhBqZuKxWqzBNUxhQTCJwrYEPY19VGjMfagskA99ZtkHcx16vF/F4XJhYnPYBYGxUZbFYhGmaopOkqcsKdZkjCLihqE7qdrul+sqAQD001Uyc3+8XIZ9arSY0BOCsgkvVRwZdpGXdunVLKKWPHz9GuVyWm24aL9C0ghQ2VRqeM16DwaCMNkgkEpLtZ5/GdQb3us/nw/379/HRRx8hkUggnU5LNQs4F2XggVcqlVAoFNBut7G9vY39/X20220cHx/DNE1R3WRFhVV0NSOvKjDzezVwoHF3Op1Ip9Njo6HoABD6XvhhqH3qdJR4CKhJh1nERU4lnWtVYKdWq6HRaGA0GsHtdgtbRA2iVPG174N6rqhOvBqs8T1UgRQ1oOX5xCoPnZ+LlM2vG3hGJhIJhEKhMTFBJm8oHkYVXzqr3PeqTWO1keI/u7u7aDabsFgsiMfjsNvtiMViWF1dlTNd7f/SeDVwP3IfdzodlMtlWK1W1Go1jEZnSvAUBnO73cLg4etmwU4xkcNkDmedc5xQqVRCvV4Xv+ZV96DKmopGo1hfX0c4HEYwGEQikRC/1jAMjEYjlMtlVKtV1Go1fP7559jc3JSz5CpfBybQrVarsM/sdrsElLQJ6jzcD2lLVZX/Wq2GUqmEarWKg4MDFItFHB8fi72fRahtbjdv3sSvf/1rRKNRPHz4EKFQSOwJ21a++OILbG9vCwu33W5PbRvEB63oqvNVJ2e5sRLIqpfqEE326LJSyDI6D1xm0Lrdrgx+ZoWKo1pI3TRNU8a5TOOFmXZwDpoqF89MsdfrRa/XQzAYRDAYBICplR1/16Bj7XK5MD8/j7t37yIUCon64KS4FMUaKpWKUOm//vprPHnyBM1mU/pY1ErJmzAQGOhStK1arUo/KRNJk9l9fV+8HColSq1w8XsqEs6yw65WaFnRUKuodDrIwqEIGhOiqjL+D2XZ1QCXZ4nKUuB5w+tBoSV+Fv4Onj/qvaGqT17ne8JqtUo/tWEYsoYqS4FUc1UoUp2py+oY17Pb7cJisaBcLktPaCKRkODLMAzE43FR6SfDaparKm8Cda04X9fhcEhymYUEnsNkGM2KMjztkFos8Xg8knCr1+tSXeX+fZX9p1ZyOXt6dXUV6XRa1IfZDsBWIdJ38/k8Dg4OcHh4OCYqeVVBcUwAkmyZFtB+k0XElrBarYZyuYx8Pi9jzmYV6j2STCbx8OFDxONxLC4uwuv1ypldr9dRKpWwvb2NJ0+eoFqtol6vC2V5Gu32B4k+VKeGWV51JAozyRy5wsHnfA2dc8MwhE9P2Xz+m5XEQCAg73FRQEAjR6XZSCQi/cCzvMnfBFxf0hicTiecTqcEdcFgEKlUCg6HA0dHRwBwrR0Yi8UibIJgMAi/3w+/3y/DtoHxSni1WkWhUEC328Xx8TGOjo7QbDalisu9z0P3bR1tZrSZEOJnZbKH/b7q/anxcnD/syKu0uFmiRL4fVADUAahDGbUiqtaFVGfO6k+riZGAYxpPpCpY7fb5UzgtVArkwDkNWrClQEyz55msyk0Q4r9sJdPTfBdl9YXCujFYjGEw2GhEhNsV6GiL6sfavJtsurNr2pvr0qnpS3i2UE7eV3W9ENBTVS6XC6Ew2HE4/GxKjmvH6uVdPxnZZ3VXlk18Gm321KpepXZrbwnaLfcbjfi8TgMw8Dc3Bzm5uaQTCYRiUTg9/slqCaDxO/3yzVYXl6W3mBWQZl403h7qNecyVayRXn9maSeVtrt+4Da4sPvfT6ftNklk0kEg0EYhiGU5cFggEqlgqOjI/FROWqO+3Va1++DBLrsRRgOhwgGgyiXy6hUKmP9DIuLi/gP/+E/oNVq4fj4GNlsVugkpmnCarWKMprX6xWVWM78Y+M7hX5CoZD8btW58fv9cLlcSKfTWFtbg9VqFUGNWRJmeFtwjUi7ff78OUqlEpxOJ+bn54UmDgCHh4c4ODhANpu91utL6lI6nUYymcTS0hLm5+elejQanQ0w39nZQb1ex4sXL/D555+LEqFpmkLnrNVqQhV8WxVfGjJWUOgIxWIxxGIxEQxTJfjV/i6Ni6H2J9psNsl6cp4cg6xZhBrAkoXDQIZBJxMDaj8/VTCZeQfOaf6TgS7PBL/fD8MwcO/ePSwvL8sM12AwOFbRVYMvNeiddIYY/PLsIFWr2+3i8PAQGxsbaDQaODo6Qi6Xey9qoR8StA9utxsrKyv46KOPEI1GEQgEYLPZxoKiRqOBfD6ParWKRqMha3hRbzp/brFY0Ol0kM/nMRgMUK/XJdHBJGCj0RirMmpH//Wg7t1QKITbt29jcXERy8vLcvaoitmkkL4ORfeqQ02mAeeMqpOTExweHiKbzUrR42Xnrdq3TvZHLBbDL3/5SywvLyORSODBgweIRCJShGHbHgNdp9OJaDSKdrsNj8eDjz/+GAcHB/jf//t/Y2NjQ8atzcp1ed9QWXZer1dEasm8ajabQl+flTVX9zED//n5eXzyySeIRqO4f/8+VldXEQgEhPnQbDbx7Nkz/PWvf0WlUsHGxgZyuZys4zSffx+sotvr9YRmzEy4Ol/P7/djcXER3W5XKrJ8jtpjy0A3Ho/D6/XC7XYjEAiIShjfj8ZddW5Y0WIPZSQSkQN7Vqi17xJcXwoPAJCeLAAIh8PIZDLo9XoiQHKdwYpuOBxGKBSSqi73HGlSlUoFlUoFe3t7ePToEUqlklBp1ODpXYMz0TjTlBVd4NxRZT+Nruj+MNTeTZW+NssB7kVQKVH8XmXzTM5YV8fPTSrv8ysfVHQOBALIZDJYW1tDIBDAysoKotHoWN+R2suo6jJc1M9LsC8pl8uh2WzC5XKhVqvB7XajWq1+RzjrqoL7NxgMIplMIhAISEVXFehSFWrZ369Wc9U1mKTTtlotuFyuMdEXXmee85zGcN3PincJtTrDCmMkEpGqDPc+9zwfagJpFqC2UqitJ81mU8Zj/VC7ifoeTNRxhODa2hpisRgWFhYQDoe/I77K7xnw8v6JxWJwuVxj7U16/78bTCYxmWxVK7ocozgrFV11TdSz1+/3Y2FhAclkEplMBqFQSGacA+dFrf39fZm2Qp912hOTHyzQ5WZiRcvr9Qpd2ePxoNfrSV+ty+VCNBrFYDCAx+NBIpGQ/iFm/sPhsATEzArT8KgB7WRPMIMwv9+P5eVlEdzI5/NwuVwShKsZao3vR7vdRqFQkIw/xcW8Xi/C4TAqlYr0B12U9b8OmDSmFGFotVoAIAfo0dERHj16hEKhgJ2dHZimKdl1Vb33XX8ui+VsnvTS0hIikQjm5uZk77fbbZmFSQd21vtLXwVqco0qtbQ57AWb1YB3UoBw0nFj0KQmPkkTVvvk+AAgtlytAi8vL+PmzZvw+/1YX1/H4uKi9JkyE83zh44+e8kmxdwYeNER4v0BQKjQ6XRaxPdIZe50Omg0Gmg2m/K382+cdgcAGE9EeDwe+Hw+SRgDkOvEXkbTNGGapiSJOed7shKmXnOKFlqtVqF7TzpawPRS394nJv0VOuMqtZgJSDVxP5nwYTsXndVYLAbDMETUs9VqoVQqoVKpiPLtdTyLLwIDTHU2NO1BvV4XgaiLRqOo60z/0+FwIBqNSmKIVGWPx4N2uw2L5UxxnJMRVGahahMZ/LrdbqRSKSwvL6NWq+Hw8FDPE38HUM8h6gFEo1EZo0jaLcfazYrPw2QykzR+vx+rq6u4ceMGYrEYotGoJF3I8CuXyzg+PkahUBDK8lWxHx+0otvv95HP5/HZZ5/h4OAAyWQS9+7dQyQSkQOPVTH+THU8aBRYsVWdSrXvAhjvWVFfR2cmkUjgZz/7GZrNJsLhsKje8qs6Nuc6023fFqPRCKZpYnt7G6VSCT/+8Y9FBCwUCsHpdKLdbkvvtJpJvi5Qs2MOh0MCn263i3K5jF6vh3w+j0ajgc3NTfzLv/yLHGQUQFCd4ne11yYFeqLRKH76059iaWkJS0tLwoRotVrIZrOoVCool8tjgbfGxaDTEwgEhGlCB8btdsPr9c6M0MtF4Po4nU5JPhK0pwycqLzMw5O9nGr/J+02E2hU3f/Zz36Gf/zHf5T+uFgsJvehxWKRCiTbAHi/cawEBZZ6vd4YHVFNptrtdqkOBwIBpFIptNttUVGt1+vY29sTqr+q8sxE1zRDTcwEAgHEYjG43W6pPPX7fZimiVarJTRP0zRxcnKCUqk0Jr5GqFVG4IzpUywWRbWZNk49k7lms3LWcm3IMPN4PLLvPB4P+v2+BKS1Wg2VSuU7VadJunImk8HKygrW1takbcZqteL09BSmaWJ/f19ax9TxcrMAj8eDaDQq/edM4JycnGB3dxeVSkVmEqtVVbUNg+wRj8eD27dvY2FhAZFIBGtra1hYWJA2PdM0cXh4iCdPnggtttVqwWKxSKuFYRi4e/cuFhYWJFHn9/txcHAgbUyqT6vxelD7cl0uFxKJBFZXVxGNRrG4uIj5+XnRQKHa9iz4PDwfXS4XUqkUfvvb32JhYQHz8/N4+PAhgsGgFA1PT0+Rz+ext7eHUqmEFy9eYGtrSxLSky1F04oPxtflzdrtdlEqlcTxmZubk4wLg1dWSex2+9jcPTVTrM4G5XtdNP9MpY7wtXRomAGNxWIIBoOiKEnK1iwpbr4pmG0m/VvNilEQjE4Ts9TXmZajUh9Z0e10OqhUKqjVasjn88hms8hmsx9MDIQG3+12IxqNCqWNzqVKR1Qry3q/fz9UFU+VOaLSb2c50OU6qNU69d4nVZ5BkvpQD1CVXsxDmmdENBrF3NzcmB2n7abd5n3GETisjrEPjokdngsOh0PYRKxq8nvDMGC1WtHtdmWEiFr9ZYuM2hM87VArq6yUq8kJtaWBDnur1ZJ1m6xYqectQTVmu93+nbaMi54/C1ATkWzjYisWJxdwvXjGAueJIpWB4Ha7EQqFREgsEAhIfyhwPiWh0Wig0WhIEHUVnNR3Aa4Vhc94f3JfqqP2+PxJPQDaM/Z6UrgtFAqJ9gXZKDzzc7kcqtWqtCZZLBaEQiFRYV5cXBQGXCAQQLfbhWmaF7JgNF4dqr/PYJcjnpgkZUGCDJ9ZSvqoaxKLxTA3N4dUKiXaDOqM+263i2q1KurKFGO8KkEu8AEDXRrmXq83ltnt9/swDEOUkB0OB+bm5rCwsAC3241kMgm32z32XqrhUWklqoomoQbCKu2Hh4PNZsPi4iJ+/etfo1qt4uTkBEdHRyKeUSgUhN5CKomu8o6j1+uhVqvBYrHIzFweFHRKA4EAwuGwHCjXSeFaFeYql8vY2dmBz+dDr9fDxsYG+v0+SqWSZI8pNPU+g0m1H4gZ6GQyifn5eckgk0pOMbFyuYxisaiD3FeE2k+tOvper1fE8FgVm0VMCkGpwSfpYhfRYFWBHNWp5+GcTCZlHvrS0hLC4bDQCYGzhCdF3EqlEjY2NmCaJur1OgqFgozyYFKOvxPAWLI1Go2KwGEikRAH1+/3w2azIZlM4vbt26jVakJ/fJsRYJcBnpsMblmFVyusdL5N05SxKFzPyRFML+tVZrDMKuJklUrtmZwFh9NutyMejyMcDks1L5PJwOFwwO/3w+l0SkV3MBhIJZ3q1dQ8YVAbDAZx584dcVaDwaC8B4UF9/b28PXXX6NSqeD4+Hhm2lPoH6r6GRxNSYHScDgM4EyQjbOe1Yq7Oic3k8nA5/PJ9bPb7Tg8PEQ+n0e9Xsfh4SGazSZOTk6wvb091pphsVhgmqbM8e33+9jZ2ZEAOhaLoVKpIBAISCJplkSS3gXUmMDv90tB6+bNm3jw4IH4okxMtlotOX9moaJL4Uzu+1QqhUwm8x2lfZ7T1WoVuVwO5XJZmFbEVbHTH1SBaTQ6G0WTzWbFeLx48UKoOxxSf/fuXXQ6HVFI5qYExvt+JqnJF2XRueHVIJeBLjOot27dQiqVQr/fRy6Xw+HhIRqNBp48eYLnz5+j1Wrh8PAQpVJJDmH1hrgqF/t9gdnLwWAg2Uv2DTFTHYlEkEqlRGGYszOvC5gAOTk5EZGaJ0+eSC+QWrUiHe19gveDy+VCPB5HKBTC8vIybty4gbW1NXneYDBALpfDl19+KRnoq5Spu0xMzg+lo89eIN4Das/mrEDtjVIDXYIiMOwZpF3gek7aazUgm5+fxy9+8QtEIhGsr6+Lcjh/z2AwkITb3t4e/vjHP+Lw8BDlclnU/FklU6u+arJUpZDGYjGsrq5Kj/vt27fhdrsxNzeHQCAA0zRlJiYrn1cliFArXW63e6ziBZyf2WzryWazODw8lDnck+NYJntI+W+uN5MKkwJWvG4vC4SvC2gLHA4HFhYWcOvWLcTjcfzTP/0T7t+/L/eN1WoVeuBgMMDBwQE2NjbQbrdRrVZRq9XgcDiwsrKCdDoNn88nIkhsn7Db7TL6r9ls4vnz5/jjH/8I0zRRLpevzB59W/AsVINTVrudTieCwaDowHi9XtTr9e+o0lIXIJVKyWzcTCaDWCyGdruN7e1tSRQ/ffoU5XJZevcnz1P1fV+8eAGv14v5+Xn8p//0n7CwsIBarYZIJCKvVyvNGi/HJGuTlH4mQx8+fIhf/OIXkmRgor9Wq6FYLF67AszLQAp+OBxGIpHAwsICVlZWxP6rtp9FGlL7a7XalfQNP7jUMBev3+/DarVKZpL9bKwOUrhCPUwnM2ycX6niZXSPSccJgLyWind0sjqdjigXss+U1ed+vy+VaTq66vvPIiYrW5M9uOphcZ0pOdzb3KsM+NVKBZ/3IUBjT7YE2wDcbrfsdVJT6vU66vX61MvETyPU/TzpGM0qdRn4bp8mMG6HX6YCe9H7qPRal8slVSxSitX5q6zSsteXjj170BmIkpmjViMZZDBBQRV5JvJisZh8RqfTCZ/Ph36/L1XQ09PTl1Y1pxWqaBgdfHU9WSGncBh7mn8oITZJVf++5110Pl9XcH05bz0cDiMajUpyjGtG28zWEuonqKJV0WgUsVhMqLSGYcjvYBWddGX2w9fr9TFhq1kBbQfvVe51p9Mp5yJZDWpVUB1/xokFTCQA50k72hm1B/qiaqzqxwJnhQLDMDAcDuXs4O9iFVjj+zF5BquJfo6g44PXX2XizQqThFDPOu5v3hOqDZpsC+Vr1bP9KqzZpczUmXR2qF4JnDkPh4eHY70qpBirqnXcsFRdVjNwL/t9hHoj8ILToFFtrNvtwu/348aNG+j1eqhWq2g2m6hWq9jY2ECpVEK5XMb+/r6Incxq1o2BHDP23W5X+nPZq6iOXrnOhltNfqiVvA9NB+Ye93q9WFlZkYdhGDK3Mp/Po9VqyTxQZv41dfnVwB4XisiQOkt2CnsSZxVqfyxwfj/Q5tMRZEsIqWQXUaMY3Kp95tFoFD6fT96Tj1qths3NTRwfH2N/fx87OzvIZrPi8KsBLj8nfw9/RqfH4XBIgEe11tXVVTk/vF4v+v2+OMnquIqrcA/ZbDYEAgEkk0mkUikYhiH7mOdztVrF1tYWstksjo6Oxvr4Xwa1osu1UsXJaJ/U68b+5suwlx8KPAdZxbt7964k1NWkmMqIoHgmCwHs93Q4HNIjyuCIydZGo4F+v4/NzU189tlnqFQqePLkiQguMSF73aH6ej6fT3qYmSSzWCxYWFiAzWZDOBxGrVaDYRiS3BkOhyLQ5nK5kE6nZWRTqVRCqVRCrVbDkydPcHx8LD4i75GL1li1NSyYkN7c7Xbh8Xhw8+ZNxGIxmcxwkRr0dcOrBk2TGghqQo1+PCeyrKys4Gc/+xnC4TCWlpYkVlAToSymzQJtGYDMhrfZbMKuZOFRLQI6HA4Mh0Mkk0msr6+jXC6PifXydcD0F/ou1QtTg0M6BzTuNDDsQWHVdzgcSrWVw5+5udV+XOKijD2NHzN3/J3AWaAdDocxGo2wvLz8nax/LpfD73//e+zu7mJvb08oKvx7ZhEXBbo8dJm9nqVA97L3gXq4ezweLC8v48GDB0gmkxLodjodoVlns1kcHx/riu5rQh0vxIOVDlUwGESr1ZrZQJcB66RA0aSTxyCSSu0qVZnvA5wHul6vF7FYDKlUSsZ5MCAjLbZarWJzcxObm5vI5XLY3t5GsVgUFgNpyurvUCm2AOSzWSwWVCoVFItFORuq1aqMq2Cg63K5xL7xb7sK95HVakUwGEQ6nUY8HhdKHyu5w+EQ1WoV29vb2N3dRS6X+45wy8uqtmqwq1YOKEyoCgJdVC24blCFp9RA1+/3i6iZCrJxgDO/JBKJjPWUszXC6XQCOE8gUfCz2Wzi22+/xb/8y78gn8+jWq3CNM2ZqlwB58wm2g6KQXm9XjgcDszPz8MwDIRCIZimCZ/PJxX0fr+PdDqNpaUlobwyED46OkKxWES1WsXTp0+Ry+UkgfcqzARV8Z39uL1eD263Gzdu3BDa+ubm5rUfNfQ69z19SrXtgT8HIFVcj8eDlZUV/PSnP0U4HBbmw3A4RKPREAFOrvN1Ta5NQh0Dqga6apWb9hoAEokEbt68iXK5jO3tbZkwoFbBp33tpsYL42KR6kk6AQMn0oVJOfN4PAAgVatJNV/V0eJBqga6kyOKJmmGDJ5JLyEMw0AwGEQoFEK5XBYHd9ak+iehZtcmxUlU2uEsUzk/JFQKFIUHKMBBZ4lDv5vNptxXlx2kXyWoSpwq5Wfy+1nF9wmu0QarQi+qarVaCWRSUp1ZzOe/jMGjqkaqZ8D32eeLqslq1ZHvpc5nnwyYr6IYldPpFAd+clQf6eXtdhvtdnts9vDL3u8iBhUFvhhcvGzNrjvU/cIAhwwbrv1kixZtyOS9wefYbLaxRE+j0UC5XEa9XkelUpHRXRRfuyp7831h0j5fREs+PT0VR586I2pCEzgLGHhfXNR7/qqYfC7PlOve+qLSYyf3Of9f/fu5TvTh1WScelao/g6TkeqsexZlmGSdxXuCxal2u41yuYx8Pi8CpWzbYezD+dO9Xg/hcBjxeFwSLzwfmfycVkxNoKuC2QS73Y52u41isYjT01Nxyr1er2Te/H4/ms0mXC7X2MgIHsickdVut+X9LZazcRBU6yQ1RVWeVAMz1TAahoH19XUkEgl4vV5sbW0BgBwqs9DM/jKo2Xluehox9ok2Go3vJA803i04wzgYDGJ+fh7Ly8tYXV2F2+2WvsODgwN8+umnODk5EZGTWU7UvAlY1eNoCbW9YrKiOWtgtcJisYiDrQa3AGQkEEdrcHYo55fzgHU4HAiHw6IWHo1Gx3r91Sy0yhyZbGeZ7C3i51CFxCb3v8oE4ucJh8OIRCJjo1soHMPs+FUI3NhPPjc3h/v370u1i8lmVpkoUnd0dCRJsUna4GSSeXLNg8EgVlZWEIlEkEgkJNjtdrvSN8oq+LSv25tA7fm0WM5Gum1vb+Pf//3f4fV6ZeQbE/Dca1T7ZrClBgf8ntdqb28PlUoF2WwWf//731EoFHBwcIDj42OhMl+Fffm+QLolgxvaCDr4p6enIgLF4kW/30ckEkEymZRg1+PxSOIgl8tJdfBNAiZ1TzC4nQUfkn8v93s0GpUkAvufmdhkMKWyK5mAq9frMvuYAVo8Hsf8/Dz8fj/u3LmDeDwOn88n7IjBYIDj42McHh5if39/rF1rFu4Nnlf9fh97e3v4n//zf+Kzzz7D4uIiPvroI0QiEQlobTYb/H4/lpaWxG7fu3cPlUoFjx49wtHREWq1miiNA9NJY57qQJcHYbVaRa/XQzabRblcFll2v9+PQCAA4EwWnoO6aaCYZSuXyzBNU95/NBrBMAyk02l4vV4kEgkMh0PJ/Hi93jHhAhpFi+VMon5xcRHJZBKtVguJREKMXLVavaQVu1xMUg0nZyoya6rSOzXeH9R+pHg8jnQ6jUwmI//f6/VQKBTw+PFjHB4eolKpfG9PkcZ3wYOajg/39UV02FkEewUtFstYP6BqDyigQ9vLSgmfp/blxuNxJJNJERS5KFlGu/JDQS6fq/6f+nvVII4/Y4CuCmGpys3snaR40FW5/jabDbFYDCsrKwiHwzAMQ9aAieN6vY5SqST0b/59k9V0dT3VANhiscDn82Fubk6CaVL6Of6GY3TepCJ21UDH/ejoCKenp3C73YjFYqI7wq+xWAwAJJnGPTjJZuC14mjEnZ0d/PnPf8bR0ZGM8NKimeftcUyoqBVAKlQHAgFx2A3DQK/XQyAQkFY5Vn7JMOTYQCbzXhe8P1R2Cz/XdU6S8vxkv3o8HpcqLO0D/XubzSa+vCpgyBGg9Xpd+nJtNhsymQzW19clQGNBSxVWKpVKkgRicWxWwPMKAI6Pj/Hpp5/C6XTizp070odOOj9ZOIZhSCLo3r17ODk5wenpKex2OwqFAorFIlqt1tTal6kMdLmZ7Xb7WIZfFQ9Rqc1UETRNE8ViUUY8kOrMmX/AuQPKvlqPxyM9YswgMdBlFogS9F6vV4Z7s5qTTCYlU8om7et+UL8MDHJ5zXi92GvERIIOdN8feGgahoFoNCqq4RxX0Wq10O/3UavV0Gq1pD9lFvfr24JVF8MwLpw/N0uH5yTUgIVrQZV8OncMGofDIfx+v9BaOXOSLBDaZAoTOhwOsTF0ENWgSx1X1263EY1G5bxQHVIGZCrt8KLeU9URVQNolSJNFotaMZvme0ptJ6F6NB1+AFItqdfrIuD1KtXAi2jLdGp5vk6O3OLemIWqiuq7tNttmKYJl8sFAGi323A4HCI0xR5xJoTIZCAdkwwGJpU6nY7YdO7pyer7LEK1RapAHu0AJ200m00RrFPXcDQ6F9WkoJrL5UIwGEQsFpPXUFD1VddZZaBQpZ+2jAH5VRYNm6xUs0+aSvk+n09sTzqdFmYUK+wMsMiAYKDLPe52uzEajeDz+YRya7fbZY40/Xe1ZY6UXc49r9VqEvRd1XV+G6g95fV6XTQYmBhT+9KZoONah8NhJJNJABjTD/gQ4zNfF1MT6KqGiHQmHrbNZlMOB+BcyIQ0FFZrc7kc9vb2ZMg2MwwMevl7AEjPEC8a56qxR4MKzLFYDIFAAB9//DFu3LghQZvH48HCwgL+43/8jzBNE3/+85+RzWa/d0zGdYXqZHJUjc1mGxPMSKVSMmyaGVFgNo3L+4IaQKysrOCTTz4RlUngjF6/vb0N0zSxsbGBfD4v1dxZprS9CSwWC0KhEG7cuIF4PI5oNCpZeDqdKmV3FtdW7ZPtdrvodrvSksJK4u3bt1Gr1VAoFEQRtlqtotPpwOfzIZPJwDAMRCIRZDIZCXZbrZYkOunMsPXE7XZjeXkZfr8fqVQKdrtdxHlY3VIrBAzmTk9PpSILnNsmh8Mx1qNHx4ljX/ggHewqMCNYPfJ4PEgkElheXpZgl6qcGxsbOD4+xvb2tpzBapJA/XoRTZ8JAjq4iUQCqVRKxCV5v6jX4roGuvybGOByn5VKJWE4MMhhYtLtdouiciqVwvLyMrxeL1ZXV7G6uipBgtvtRqPRQKlUwsnJCUqlksyRpvjadVzTVwX3lGqH1JFmhUIB1WpVhOsODw/lucCZHbuo3/PevXvwer3I5/Nit151//LeCAQCwr7i+5OZWK1WUa/Xr2TCVGUfhEIhoSavr69jcXFxjK5vt9ulRYXJNzVAJiuIj2KxCNM0xxIWTKSRastWxEQiIaNzeL2bzSZ2dnbw5ZdfolKpoF6vz+z9wbYHq9WKnZ0dtFotuN1uSax5PB7cuXMHd+7cgcfjkTnUwWAQ9+/fx/z8PLLZLOx2O46Pj1EoFLCzsyM9vNOyrlMT6ALnBomVJwBjlVn1hmflkH0Xp6enODg4wNbWlggEkJLwQ0q4F4meOBwOZDIZJJNJxONxzM3NIZ1Oy8w19o3dunUL7XYbh4eHQo+4iobpXYFOPq8JcCaMFAgEEI1GUSgUxHhNy01wncADIh6PY3FxEaFQSITbSPUpFArI5/MiUjJNBumqgG0M6hxLOv2qbZpVW3BRBWUwGAjlmHoHAIS6Fg6Hx0YNUQk1EAhIzzltL6n2atWA1UgmKcnGqdVq8Pv9qNVqMpuSY40YiKtCh6oCMCmOrLjQGVOTGrzWV0nQTa0mMZFAB57J4WKxKMqyFwVMk325F9kQVYiKVERWdLiGqlDYdbZDqoYFAAmkLqqCAxirUi0sLMg+5vxcsqOsViu63S6azaYUBq5S0uV9Qu29VxNvvM8p3mWapoxOOTk5AXAu/jUcDsdm23L/ptNp2duGYchzf8i3Ufus+VpV6MpisUjLwFWl1aosGJ/Ph2g0ikAggPX1dTx48ACGYWBxcVESxKrAlKqZwLWije31emOzjlkFZ1uJWriibVOFrpjMLBaLODg4kATlrIIJIAAolUqoVCri25BNNRgMxHbzWrrdbmQyGcRiMbjdbhwfH8u0nMPDw6krnkxVoAucZ4bpMPh8PsTjcRkrxPlPdDgGgwGq1aoYLGbrVKrVqyy26pixlN9oNCTTWiwWkc/nYRiGbACHwyE051AohHg8jtFohFqthlqtdiUN1JtCdXpUqqJaQWdmTeP9QFUBZhWK4hatVgu1Wg3ZbBbZbBaFQmFsNt+k06rxcjDQUSt9pBCSWsVs/CwIi7wMtAFMXDYaDYxGIxHuAs5HRfAAZaWWzA/VQVGrinQ+6bCqPbtq8AlAlMfJFuK+fxUHx2KxSI+w3+9HOBwWO6YGaleNdqvaCVW8i2dgp9NBoVAQXYyLKJRqEPEyqE49aZ+8pirlVm2huErr+DqY7Gn+ofWkPQGAZrMpfXDHx8cihgmcBcSDwUAcfN430zDqbhqgtquVy2UAEHFSOvpkFfArwVF8zWZz7Fy1Wq1iV3jOqsmbSaj96hQXc7vdSKfTiEajiMfjEoAVCgUUCgWUy2VhmlyV+4F/YyQSwfz8PDweD9LpNObm5uD1erG8vIxYLDbGjlFtO+n8vV5vrB+aSYp+vy89oSr92W63S2WXdH/gu9NX1P5elXWlcQaVedLr9WCxWFAsFrG9vS2Brs/nE1+Tiv1zc3PweDzodrvY2NgQxsq0rO1URh2s0rpcLsTjcdy/fx8AUC6X0Ww2xwaidzodUf4yTRO1Wm0sY/cq/SmqsAYz++wLME0TlUpFhoqz8sBKGdX6VlZW8ODBAxQKBWxubqLRaEzNRf5QoJFX+ym4BuohrIPd9wN1ViWrYez1GgwG2Nvbw1//+ldsbW3BNM0x8QDV+VR7TTXGQfYHaZ9U4HW5XELDLJVK2N/fR7VanWqBhvcJNWHZarVQKBSQy+UkG8y+RB6WqVQKt27dEoeSNGSO+Oh0OuLcsHeXVV/DMOBwOMRJZWBNddVAICD91Jx7a7VaUa/Xx5xafu7JeyIcDuPevXtIJpO4efOmCGSplLrJ/txpbs1QhdR4LZjQZcWxUqng6dOn+Oqrr1Cr1cYCAxU/9Pep4mOcksC1Gw6HaLVaqFQqqFar4nReR7szGeQCF6+d+jN1dMrJyQnq9br0JVI46f79+7DZbOh2u/D7/chkMmi1WlIEmKaqymXi9PQUxWIRGxsbIqgDQNaS7QuNRkP0XIAzG1Aul3FycoJutyvsNAo+AmctQQx2J6dOqO9DBkU0GkUkEoFhGHjw4AGWlpakQGKaJg4ODvD06VNpt5i0UdMKtYq7traG//pf/ytisRgymQwWFhbgcrmkUKRWcXu9njDMcrkcvv32W9RqNUQiEcTjcUki8zxhmwvjA9p/ajycnp5KRVdtqaC4XrPZRK1WQ6VSEfutcQaefxwX1Gq18PjxY2SzWQSDQWH2eDwezM3NIRQKIZPJwO/3o9vtwu1249tvv5XxTdOiVTKVEYdKZ/J4PIhEIvJ/LpdL5vnxOaTs8Oc08K+zwDwM+DrSR3q9HqxWKyqVCsrlMlwulwTCvKkppkLREwpFzBIme7VU2iIAqcLr0ULvB6q4jBrw2mw2SfrU63UUCgUcHx+PVVAmFWmBV6vWzCLUzDz7gtjqoFbD6vW6VCdnFapzQs0EwzDGgknuW/Yjkm7GuaJqRZe9RExgqraEKs5ut3uMnghArg33c7/fHxNEuuhzqz2oTqdTnC4GzepooasYTLxsFBP/nl6vh0qlgkKh8MaCder6qvRvde3UvslZEcV71b+RFV3VF3E6nVL1UxP+w+FQEpys6F7XyvjrguvACR52u11o4wDGKn2TLSes6LbbbRk5RFE90plJmbXZbGO2g6CN4/nMfl+yAFOpFNrtNo6OjkQkqVKpyLjKaQgUXgXq3xkIBLCysoJMJiOBLlvWVDvMQIiMs2KxiP39fZimKVVeJjsZF5RKJVSr1TEaOdtieC2BcREyvl5tVZm2quM0QY2DKpUKWq0W6vU6isXiWC+6qpw9Go0QjUZF1HCafP2pCnS5IdUeN1JE1MZ2Gh2Oc/D7/fJ89hyq7/cujP2kMiRBR4zZVIfDgYODgzFlzlmAalAucmTZ+8w+FFJNZjkQeFfgGvv9fhF+oEo4q06kd/p8Pvk/n88n7QHsMVKVmNWKpD4MzqD2g7Jfn8wOYHymKtVqZ8UGqOCesVgsaDQa2N7eFiXRRCIhhyDXLRqNAoBkgQ3DGKMFU12flRI6lLQpzO6HQiG5BhTDoy1m0EB7xBmNnO3N4JnXl7S4VCqFhYUFzM/PIxKJSLBdrVZxeHiIYrEotOyrhIv6benst9ttofcxi/82769Sl5kcZuX44OAAhUJBdDmuG+j8A+OiXW+yX1S2AZMV9I+ojMrKFoPdWfJDvg8UObXb7VIUYUtcIBBAvV4XQSgGQ6enpzBNE1tbWzAMQwIydcRTrVbDaDQSWj59GgZ1VqsVsVgMqVQKbrcbqVQKyWRSfi8rZxTzIUVd1Qu4CvB4PEJLXlhYQCqVkoor7T3/Js59LhQKqNVq2N7eRqVSQbFYxM7ODhqNBmq1GsrlstCRmYgjY7Df7yMUCknSUu3TVTGZuORzWfF9WcJTY3ySSq/XQ6PRQLVaFRvOqjztm8/nk/F7w+EQjUbjkv+CM0xVoAuc90zwgLXZbJKpZ5DUarUQDoclqN3e3pYbgEYHuJh6+boGX+X4q3Ro3iA8cCKRCNbX1xGPx7G7uytUujf5nVcRavZsMpPMLCY5/nR4JunlGq8PlUobjUaxtLSEaDSKZDIpo1sooc/ntFotCSCsVisymQyWlpZgsViQz+dRLBbRbDaxvb0tzoDOfJ6Ba63OgQ0GgxIoqaMLaMNmFdwvxWIRX3zxBXZ2dtDr9XDz5k1JdrEiMjc3h0QigXa7jXA4LNT6k5MT6U8klY9jbwaDgWSOHQ4H4vG4sH+YkGQC0ufzic0m/TkQCMBut6Ner48JWQGQ3jLDMHDjxg3cu3cPCwsL0lc2GAxwcnIiFMNSqTRmx6a9mjb5+UglZvWKiZpms/lWdGL+Hq45s/1MXpCqaJomTNOc6jV7XaiBDp11NVGjfv2+95gEAyy1Oshz1Ov1ytxQ7umrFCy9L1BJnL2etVpNevSpetztdhEOh4WNU6lUMBqNcHx8LLNaGaSqibVisShrz+CYfiGv/Y0bN/DTn/4UhmEglUohkUhgNBrJKKNarYbNzU3s7u6KEBXn/U77PcE96vf7cevWLUQiEdy9exerq6tCPeY9T/pwPp/H//2//xfffPONJBJIJaZ/wtdN6gkEAgFpWeGcXZ4BFNQDLrZxrKrz/QDINdO4GPT92D+dz+fl52TocM3JUmAbUblcnoq1nbpAFxifradmMElVoGIns5eqYh03PbP3kxnN78tufh/lRKUs8jOqz2cPmEqLm5VMqnpov0zlmkaLjqm6nrOwRu8LNNyq884+OIr1qI4RnU3gfA+Hw2HE4/GxnkMyKVgV0DiHSkPjAzjf/xfNkp5FcH/1ej0JYuhgco9xb7JP1G63S2XPbreL5gKDSzpKHBOkjqJQqVLMMFOoZ7LvloczBUwYFLAqQ9VJv98Pv98vjhWruax6ckb7VUxoTLJwVNEXJsLetsdKDfbUmcesFKj9kdepV07tQeTfDozTkYlX9Uf4s0kqLH8P+xMZHFzUkjLLOD09HROdYiGFZ53L5RKGDhlQvNdZxWWlXO1tp2gPnX3VtvE1gUAAsVhMWtwikYgEdfws7A++aMrININ7TNUG4Vx0t9sNYLz3s9VqodlsitgdRzsxsUC/X/URWRxhS5bb7f7OzHMGxCp74qKEknpPTt6LGt/FpF/PM2HStqsJCVWwcRow1YFuv99HPp/H5uYmgsEgVldXZah0KBSC1+tFq9XC/fv3kclkhB5HPjlnkE1m2r9v8dVDxOv1SoY0FoshmUwiEolIJmjyM6siBHSeZkkQYjQaodVqoVQqATijCk1WwChGxYNY49Wh0m+4R+mMezwe3L59Gw8fPkQwGEQ6nZYDmwq1mUwGDx8+xNLS0phxCofDQh01DAPhcBiVSgUnJyfI5/Njjq/GOdTAlmvEDD2dlVm4778P7I0rFApoNBrY2dnB8+fPUS6XkUqlkE6nxR6w783v90v/Mysx1EogpVClJgNniUZWZt1ut4x8CoVCSCaT8Pv9Y60xtEfdblcUnwFIgsjj8SCZTIqiJHt/mdFuNpv45ptv8OzZM9TrdVQqlSsjpKSypjiGr9PpwGq1SgvQpLjWm+xjXksm4CgGxp7TRqOBYrGIbDYrc1+vA5iAj8ViounBpFe32xWRNJWa+kPBrtPplGCMirYcuaUmbFit4nMBSOvKrEPtCT86OsLjx4/h9XoRiUQQCoUAAB9//DGWl5dxcnKCZ8+eoVarCbuh1+uhXC5LrzRwXg2cm5tDLBYbC9TYw8uK7vr6utDMKdK3v7+P/f19lEolmKYpYxmvgh0BMEYXTiQS+NGPfoT5+XmsrKzIvqS6cavVwjfffIMXL16gUqng0aNHMuJHnbt6kb9OISu3242FhQWk02n4/X7cuHFDRtyQycB9P5nEU3UJvF4vDMOQ+/EqJio/FHg2k3a/trYm9wz1AJhcmNbk2lQGujQU3W4X2WwWT548QTweRyKRQCaTkeotOeKj0UgUCcvlsjRLq6pfPxRs8uKoWVKOk4hGo8hkMpifn5eDZfImUgW01D4+tVftuju9w+EQ9XodJycnQnVgZptjmNg/RGOv8ergHqUzY7PZEAwGMTc3B7/fjx/96Ef49a9/DZ/Ph0QiIT24rFCRatvv9yX5oO7309NTRKNRmSe4sbGBvb09yThrjEMNchkw1Go1tFotXdFV0G63kcvlYLVaEQqF8OjRI8TjcQwGA0QikbG+KYfDIRUBqjiy55nVVDpNamsInXuLxSKshnQ6jUAggFQqBb/fL88djUYIhUKIxWIYDAaIx+OoVCpwOBxIp9PiMIXD4bExcv1+H7lcDl9++SVM08QXX3yBb775Rhy5qxRMsHraarXkYbPZxuYJv63GBa8lk2d+vx8+n0+Eyer1OvL5PPb39yXYvurgXvb5fFhaWkIwGJR+Q4pHqfv4++ipk4JoHGuYTCaxtLSEQCAg849JI+z3+2NVr9FodC3W9V2APmC73cbu7i6++OILRKNR/OIXvxBdi2g0il6vh42NDfT7fUnElMtlsTuFQkFar9rtNgzDwOrqKmKxmCTa+JXqv9TOAM5aOSqVitCVHz16JO0ZnU7nSmlisCDkdrsxNzeHjz/+GLdu3YLP54PL5ZLiR7lchmma+P3vf49//dd/RavVQrFYRL1el8Tb990HFCs0DAMrKyu4ceMGAoEA1tbWEI/HAZy3yjDo4vmsflYyeXw+H0KhkPScqkxNjXEwQeB2uzE/P4979+7BMAyZ5X1RgmLaMJWBLnBeLWm326hWq3C5XN+hvKmD6C0WC4LBoGTmmDGezE7zq3phVIqEyt9nYzWz0aSrTJbk+b6zph55EVgpYGYSGK9Azpoa9bvCZDWX1BDuUZVi6fV6x2aVsj9Gna1LuhYTMOr9oY5ImHYD9qGh2gr1cFTpyldxrur7BCsprOSZpim0ZPaAqjNzJ+0w964685X3gUpJZqBMCiJn/rG1ZfJ6qBViVmA4LsrpdCIQCIizxqQF+69JWaYa7lUKcoFxOtrkOBR1LZ1Op1A+X3Uv877gCCPDMESwjYIyPCuZRLsuSSGVahkIBGTuss1mkyCUTK92u41utztmfy96PwAiHkhqLWeMqpRoVo51Avnl4Fq3Wi2xQ0ya0Qaw95n+Hn9GAR4+yEpzOBwwDAPBYFDO2MlAlz8jU5H97/zKEYBX7czgWpFZxvFBbDFU6cpM+JimKYnhSYVkYLzoRFvEJD37ctmny9ZFnr+qHwOcC9uyhYaP62BrPgRoy9jyxgf3s+oDcd0vGu962ZjaQBc4yzrv7Oyg0+kgnU4jnU7LTUR1N5/Ph0wmg263K5nURqOBzc1N7OzsyGHKA4DVFlJY2NOiKheSbrW6uoqFhQX4/X7cvn1bqsl0yngjj0YjVKtV7O7uolwuo1gsyu+cpov9PsFNXqlURFBBFQZjdVtVp52FdXmXoBMVCAQwNzcHn8+H5eVl3L59G4FAAOvr64jFYmKY1N4t4Fw5nPuyVqsJNbRer6PT6WB/fx+Hh4cwTRP7+/si4HHVHPn3BR666sB7i8WCZrOJfD6PQqEga6YP03PQTubzeXzxxRfw+/1S1TAMA0tLS0JjpnMJnLMXDMNAOp0Wh5MOC+8Jl8uF5eVlpNNphEIh3Lt3D5lMBh6PR2b38nMAGDuMOd7DZrMhEAgIvZaPSqWCvb09NBoNPH36FJ9//jmq1SqOj4+vrI1Xk1qsVLHnkOfr6uoqWq0WqtUqjo6OXqkyqCY1w+EwHjx4gHQ6jZs3b0rSgLalXC6jXC5PnVP0plB7xZPJJH75y19icXFxzAlncMNqoWmakkiYTCyqVHjS8F0uF+7cuYNkMgmPxwOr1Yp2u41WqyWVx1wuNyY4qDGOfr+Pw8NDdLtdRCIR2O12FAoF2fdkb/j9fukjJ3WZZ6HX68Xa2hpWV1cRCARw8+ZNxONxSUKrokfUvmCA9/e//x2PHz9GrVbDxsYGSqWS9A1flUCX/m8wGMTDhw8xPz8v1VWqLDNBmM/n8e2336JSqeD4+Bjdbvc7ia3J4NZisYwlyX7yk5/go48+gs/nw/z8POLxOFwulyQje73eWNKx3+8L07NSqQCAJBUKhQKOjo5EMJLP1TgHzz6Hw4Hl5WVhLKytrSGRSEgrBQBJXDIRnM/nRcBtWuzPVAe6g8EABwcHyOVyyGQyuHv3LpLJpCidMqPDXq5AIIBMJoN2u41EIoFgMIhutzvWN0dj0+12pfHf7XbDMAzY7XbEYjGhfd6/fx+3bt2SWb58jlp9AM4pu4eHh8jn8yiVSnLDTcuF/hAYDoeoVqs4ODjAYDBAo9GQzCcDXHWGojYurw618d/n82FxcRGhUAh37tzBT37yE/j9fiQSCYTD4TFqM6tjAMYUlKnsSIfr+PgYrVYLz58/x+bmJjqdjgRtgKb0ELz/WVXhodxut1EqlVAul8daJvS6nYHrwFE8TqdTKqLBYBCDwUCqVFSxBs4DXdLx7Xa79EJ3u10JfD0eD1ZWVrC0tIRwOIz19XWk02mhHV80coJfJ1tOSC1lUrRer+PZs2coFAp49uwZ/v73vwtb6KoKKKl/N51E9payZ3lxcVH6GUmrfBWw9z8UCuH27du4efMm0uk0XC6XqMAz0KXS6nUKdO12O+LxOD755BPcvXtX9jAAEeLpdrs4ODjA8fGxJODVShOvD4MftlE5nU6k02mhydpsNvFxTk5OkMvlcHJyIhVCbYO+i8FggGw2i5OTE7E1pVJJ1IJjsRj6/T58Ph+AM60R6gSwVYV0ZdI4l5eXEY1Gv8P0YUKJs1+r1SqePHmC3//+98Ju4Wiyq+Qr0q/w+/24e/cu7t+/L+OE2DLFRFqxWBRdhkKhIIGuqpcz2UdrtVphGIYIeD18+BD/9E//BK/XK0JXvEf4VZ3JzUQPEw70TWu1GkqlEo6Pj2VO8VW14e8TvBYOhwPz8/P45JNPEI1GsbKygmg0OibCqY58ajQaKJVKKBaLaLfbU7OnpzrQ5QYGzoxNuVyWLDpHRqgBAGkjPGRjsRh6vR68Xq9kbtxut2R+OH9XDXQjkYhUi9krxgZ3UljUau6ks8D3nsUDRqVwqsqBF1HG37b/a9YwuV6qmJSqvKk+f5JOwmCMwi/sG+N4FLYJMBE0yxT8l0FVrnY4HLLmas/dVaSgfSiQUQAAzWYTlUoFp6enODk5QSgUEhoa5/CptGCO72C/G0dmORwOuN1uRKNRhMNhBAIBGTPBYICU2YugOkw8nDm3t9vt4uTkRJIYTI5eF6ot/9ZWqyVUcPaxUYSRiQlWaS76u7nGVJh1u90yfoUMKa4zR7dUKpWXitBcdXANyRRTbQVw1r/MRPxgMIDH45GgVg0SWCkhVZBaChQPYyKyWq3KCK5qtSqvuw579H2A/gjv83K5DAAipElWDvtDI5GI+DPD4VDm0VNgjddXrc6rflCpVEI+nx+j7tInvWrXSGVtUNiJgpisYE8KEvE1k68FIHZDFVKz2WwIhUJSdGLllv9Hqj6ZaK1WC/l8XnyXVqs1pgzMBFuj0ZDRdWp7lsZ4W5aqQB6JRBCNRkV8irYegFTta7Xa2FisaVMNn+pAFzg3SKZp4s9//jP29vawsrIi2WHSF3iDhEIhkYbPZDJyKHDxaVxokIbDoShwqv0Gdrsd4XBYMn7sEVPHBjB4YLBwcHCAfD4vdKTrdnj/ECh8Ua1W4fF40Gg0xIEiPY3rf9WGoU8D1GBX7a1TxyWwyuVwOMaMUKfTwc7ODv7t3/5NaIiNRkMcT1YUmPXk/aFxBh7cPp8Pc3NzIlLHbDEFNyqVCrrdrrxulu7/V4EqPJLNZtFqteB0OrGzsyOCMA8fPsTi4qLYZYr/zc/PI5VKYW5uDjdv3sTp6enYfPVkMilVL1IQVaeLQbbaC0Y2Dvvk9vf3ZY700dERGo2GtGOwx4zB8FW+tiqFu1AoYHNzU5IEsVgMsVgMn3zyCW7cuIEvv/wSW1tbkshhXylw7sBSETUcDuOTTz7BwsICMpkMPvroI8Tj8bHxUNvb2/j0009RLpdxcHBwbc6ByXEbDHRZ5VX1FdiXnkqlJBGpBrjqWUobzapioVCQalShUJCq4NbWliQy8/m8OPvT5HBOCxgEtVotbG1t4eTkBIZhIJfLIRgMjlHF5+bmsL6+LucqhRwzmYxU1h0OB3q9HiqVCra2tlCtVmGaplB1mSjrdDrY29uT+aKv0/s+LVDH/QQCAaTTaSwtLUlRiIEoH/TLyUxQZ90CkPZD9veGw2ERn4rH43C73VhZWZFiVL/fR7fbhWmaePToEY6OjoRRSZ9msl+Xa00bdnx8LOrn+v44Z6PQXmUyGayursLv9+OXv/wlfv7zn4+xqoDz1opSqYQvvvgC+Xwejx49kgTmNCURpj7Q5c3SarXw4sUL7O3twTRNfPzxx9IPwMBUHQJtGAbm5+cla6fSNidVmFkRUOXH1cwTManMxvcmdahcLqNUKonAyrRc5A8JzpdUq9tqoKuK9WgD8+qYrIJzf5Gyoz5Go5HsWyZjms0mTk5O8PXXX2NjY0PoWAyW+Tqd5Xw5mOmMRqNCqWIVjDaAvVx6/S4G7/nhcCiJAavVioODAzidTkSjUbEXXq9XMvqkg6oCUsC5SI/dbhcnSq0QqPcK7T8rNUxWNptNEZd68eIFdnZ2UKvVsLW1hUqlgk6nA9M05Qy5LoEZ73fOseS6sif65s2b4lCGQiGUSiU580g15DlJUchkMomHDx/iwYMHCAaDWF5eljEeZJJwdEulUrl2Z6Xaa8iASO3VpDM5Go3g8/m+w3JSA10mWTieq9lsjtGcO50Otra2cHR0hGaziYODA0my65FCPwzuZY7R83q9qNfrMAwDqVQKd+/elWru6uqqtKx4PB5h9ng8HkmgDQYD1Go17Ozs4Pj4GMfHx9je3hb7wn50VW37KkLdxx6PB+FwWAJSClwyiQhA2kfYJshkAe8Jri8TndRWICuTwrBkhdDnrlQqePbsGZ49eyZ9/9SFmfSP+P0kw1BjfJoHmauJRELGCN2+fRtra2vyf/R5uI71eh2bm5vY3d3F3t6eJOSmCVMf6BKsBFgsFtRqNezu7sLpdMrgbQrE+Hy+7wxLZxV2UkCKD954Kh2UUHtm+Bmazab0brG3cX9/X3rOZlX1cPKw5s+A82vAQ59ZIX0Yvz649ywWC6rVKur1OgDA7/fLPcLeaFZpK5UKyuUyWq3WGMNhcvzWrLEQXhWqmmowGEQ4HJbWCeDsmnAe4Kze/68KBknqXuOatVotnJyciGNTr9dFKZw90QBkf6uJznq9LlVcQk1uqr1bqq0ii6HZbOLw8FBm/jabzbHRQdfROVIFRCiqRtorbXQoFMLi4iIAjDnsPDc5ozsajSKRSCCRSEjSgdRoVm8bjQZyudwYbfO6rKlKO+50OqhUKigUChIQUW2Ze10NCNi3SNve6/XQaDSkp7FcLiOfz8tzyOIpFArSbqLqglyXNf0QUG1Qp9ORc5VVcdL5GdixKMLWCF7vXq+HYrEoc3FJI+cUiklF/qsK1cdj4E4qvVq8YKGIwbDdbsf8/LysKe0HxTVJBff5fHA6nbBYLHJfkGV2enoqdrlUKiGXy6FSqYjvTcaOqrpMez/p58wyVGV8Jh8Mw0AikYDb7cby8rKMU/X7/WNxEW16tVpFq9XC8fEx8vk8isWiXCdguthsVybQVQ+Dg4MD/I//8T/kAP7kk09kiPra2ppcOFKNmd0n1OCLX0lHVgNk3iSqUm21WsXGxgbK5TKOjo7w97//XdTk9vb2rmzfxbuCKi/ONVCVfym5bxgGOp2OpjC/BmigWW1yu92ifBgKhSTxA5zTMmu1mjAh9vb2cHx8DNM05TqpDj9/h8Y41ERZKBTC2toa5ufnkclkhFZbr9fl0CVlUK/lxeCeUynFDFz7/T7+9re/4cmTJ2NCUurc58kqrt/vH6voDodDcfyZgGCCkuJHrMDwXKEtUpOVdJwm75XrhH6/Lyr5c3NzuHPnDhYXF6Vi4/V6cevWLfy3//bfUC6Xsb29jadPn6LT6SAYDIpK9f3797G6ugqv14tkMgm/3y9Mh1arhUePHuH//J//g0KhINTNqziW6fvAatNgMECxWMTjx4/RbDYRjUYxNzcHh8Mh/YQApALW6/XG+jc3NjZQqVRgmiYODw9lL6tUSz7Y68l+Xva/z6r/8aZgm0+5XIbNZhPaMXvVSdlkqwSAMd9QDWhZhWfvOwPCySDsqoJrBZz1MlM7h/34qmK+xWJBMpmE3W5Ht9sVxofFYoFhGFIl5OvUvd3tdmWCydOnT/H06VP0ej2ZVc//J9OBdFl+Rn696PtZhspWNQxDkgwLCwv46KOPEA6HkUqlsLi4KBRynr1ki1SrVTx69AjZbBY7Ozv47LPPcHh4KEmIaVvnKxPoAueBp2maeP78Oex2O6rVKiKRCBqNBjwejxzSpA4xcGWgO1npVb+q/69WcHljU1Usl8shl8thZ2cHX375JQqFghxgs+7gXmTIJwUIOJ+R1UeNVwP3FWk7drsdpVIJpmkCgPQOMtvKjGipVEI2m0U+nxcF8snq7Szv2VcB7QJFj1i1UnsPWQUkbUev6fdDXR+137zdbgOA9MSRCcLv/X4/otGoOEjBYFACXY4GYV861fV5ODPAUntNVcr/rF2z4XAotGyr1SpCRgBEWTwajWJ9fR2NRgM2m03o3NFoFJFIBIFAAA8ePBhTGLZareL0dzod5HI5PH78GNlsVijM1ynIBcYruhTHoZYIxXQ4Xgg4602k7WAPZ6FQwNOnT3FycoJyuYz9/f0xu37R79R4Nzg9PRXbwzm7FotFqrNM/vCa0o6wtYH+3yy0/qhKuxxv5Xa7x/52dUoEW04oUmez2RAMBmXeLv1A1V5T4LXZbGJnZwdfffWVjDujUvUs2ux3AfrjTqdTxB3n5+dx9+5dxONxRCIRJJPJMQFetZLPFpTd3V0ZSZnNZqe2tedKBbqEGoCapont7W2hZZK+xl4K9mtN9t1yZIU69JjvrQbU1WpVqgHdbhfVahXPnz+XKi578mbVUfohTCYTeHO5XC4Z5K3xeuAeZSBLQR+KsKm0cDr4FC1RZ/XpvfpqUHsR1R4hfqXIGkW9NHX57cBqr+o08qGqjNvtdnQ6HTSbTemZo7PF0UBqRZe6AZMD7Wc5OcmzlFTZw8NDPH/+HIFAAKurq2PJBYfDgYWFBUkgGIYhcy79fj8AyAg/ft3d3UWtVsPz589FbfYq9yf+ELh3G40Gtre3UavVkM1mkcvl4HA40O12JZiiiFq/30ehUEC9Xpd+afaMT44Iuq7rNm1Qz8deryc2ptPpiJAS/T4GfLNiR2gz2eqwvb0ts6NtNpvQlNlGQjvMcT+s+LKNimyE09NTaaOgr00BLwoDqqOJtA/z6iCrknpE6XRakpQrKysIBoOYn59HNBoVlXz6O4x9yDxh+8rTp09FgFcVaJzGa3IlA11m0ywWC7LZLGq1mnDMQ6GQZPpZcSGlQhUp8fv9YyOKGByQftLpdLC5uYnt7W3pMWLvTK1WkwtPlTctrnSGyQNZpYPzZvN6vTAMA4PBYKwfWuPVoPYdUu2XARjVEHkAU6Tk+fPnooKtacqvDybIGESxd5RBVLVaRalUknEHem3fDjwwmQxTs/6cP6kq3E6KB072ZfGhJiQ1o+Gcbst5l1988QVM08TCwgL8fj98Ph9cLhdSqRRGoxFSqRTu378/dtZRpA04uzabm5vI5/M4PDzEH//4R2mXODk5ufZjb7in8vk8/vCHP4hzSeqf+rermiFq0MRkjDoZQuPDQk0mkyY7mfBUH9eh9/Z1wL+3WCziD3/4Ax49eoSlpSVUq1Ukk0lEIhHMzc2JwriaqKQwVblcHhOoIwtiY2MDjUYD9XodpmliMBiI0COTn/qeeD2Qpuz3+xGLxfC73/0O9+7dg8/nQzKZhM/nk15qVbOIqsq5XA7VahV/+9vf8PTpUzQaDRwdHcE0zTF1+Gnd/1cy0AXOe1CYFQIAl8uFfD4Pm82GQCCAQCAgSp6BQABWq3VsRhd7ZtThx6RKtNttHBwcYHt7G+12e6zR+rr1Fr1PXJR5YzVG7Yue1htkWsH9z2oMHUjV2We1hoHZtEm+XxWogRaDLya62DfHa8DroPFucFGG+FUq5pMsHY2XQx1nUyqVRHCHe5uqyhaLBT6fT3QAWKlRdRm63S5qtRpKpRKOj4+xu7uLw8ND9Pt9eb9ZAJPlAL7DWpqc3jD5vcb0QGX4aZyDdpmK4LVaDXa7Xfxv6llM2mrVN2m1WiICWCwWpS2Q6skc5zZrSYT3BVLvfT4f0um0aCpEo1ERy1Pnfas0ZQqaHhwcYGNjA51OR66ZmkCeVlzZQFcFF5gHLTOlrBi2Wi2hUZCq7PP5kM1mvzMAmQ5rv99HNptFoVAQ+ptKmdB4ObjxqbSp9kpns1npZ6Hjo9fz9aAK+VAFOBQKydgD0oUoVNLpdGSQ+qw4mu8SdHZGoxFKpRK++eYbHB0dIRAIIBwOYzgc4vHjxygWixLs6j19udDr/+pQq+elUkkqtAcHB6IYzFn1qlJwpVIR4Z1cLiczXHd2dlAqlVAsFlGpVCQxPKvXRGU3qf+e1fXQuD6gaNTp6SmOj4/x9ddfY3d3F8FgEPF4XMbRqIq99A85XoyaI+x1Jl2ZtGbts7w9qFPEAhPp4gCkBUhNXDLJ0O12cXBwgIODAzSbTRHiZeJSbS2aZly7QJcCR7xpJiltk3PuVEoc30s9+FWlPE35/GFwrTiUe2NjYyyLt7W1JTx/8vo1Xh/cgxRHikajMlCdgW6r1RLxk0ajoRWu3xA8mC0WC3K5HP70pz8JvYfUrGKxiGKxqLP/GlcOtNndbldoxoPBAM+fP8dwOEQwGMTc3JxUdS0WC05PT7Gzs4Pd3V1Uq1V8/vnneP78uczdpWI11az5e2YZs/73a1w/sJXHYrEI85HCgaTrq+Mk1X5mVfmeAS11RdR5txpvDxb5OBJLncfd6/Vgs9lQr9dRLBbRbreRzWaxtbUl32ezWfEpmci/KkEucE0CXULNlOob5HKgCslQuZA9LQBQq9XGZrhehZtkmkHj5XK5vlNx4RqrY7L0er8ZaNA5P449LNzb7N+/KoZfQ0OFmqBUaYVUdPf7/WP90oPBQKovnDmay+VEuIfVAn0vaGhcb6gza6mdQ+0EddIGABn5pvY1E9pWvF9wfTn3udVqwWazSaBbq9VE1LdQKCCfz6PZbCKfz6NQKFxpLaJrFehqTAd4I3HeoqqgWqlUcHJyIrPPruJNM01wuVwIh8MIh8Pwer2w2WzSm0t1VK6zXus3h8oaabfb0h6hisnoIFfjKoMspuFwiEKhgL/85S/Y3NyEx+OREU7qc0lP7nQ6ODg4EGHGae/X0tDQeD9QgykAY5XayX5nnXj/cKCiNWnJVqtV2JbqODgWoqrVqswwrtfrV75QogNdjXcK3gjtdhtPnz7Fzs6OBF40eKqS5FW9caYFHOgdiUTg8/mk/4L96uw3n2bp96sArhvpVi/7fw2NqwoGuoPBAIVCQZgLwMXiXqrzo7ZF6HtBQ2N2QT9DFcy8SIBN48NhOByKwJfFYsHR0ZHouZAJSL+RRRGVmnzViyQ60NV4L2CVi47QZCZP491AHdyuUpXb7bYYNj3j+d1Cr6PGdQbtNZXFJ22HVgvW0NB4HWg7cfmgXQcgo7NUDaPJ4PY6QQe6Gu8Fk83q1/HmuWyMRiNUKhVsbW3BNE14vV7EYjF0Oh08fvwYX3/99Vh/hb4GGhoar4KXZfG1/dDQ0NC42qAvyNYrlfF3HW28DnQ13ht0X+j7Aw1So9HA4eEhGo0G0uk06vW6yMB/8cUXqNfrKJfLWglYQ0PjtXAdHR4NDQ0NjTNc18B2EjrQ1dC4wuj1eqjVarBYLMjn8zg4OECr1UK5XEaj0UC73dZBroaGhoaGhoaGxsxBB7oaGlcUpC5vbGzA7XajWCzi0aNH6PV62NjYwOHhoagva2hoaGhoaGhoaMwSdKCroXEFQRXDVquF4XAIq9WKUqmEzc1NDIdDVKtVNBqNa6GYp6GhoaGhoaGhofG6eONAdxZ43e8bb7KGet3fHm+6htO09vwsqoAAe6KnWVjgOqz9VYRe98uDXvvLgz5jLwd6z18e9J6/HOg9f3n4oTW0vukb1+v1N32pxv/Hm6yhXve3x5uu4TSuPXt0TdPEyckJ9vf3cXh4iFqtNpUDvq/T2l8l6HW/POi1vzzoM/ZyoPf85UHv+cuB3vOXhx9aQ8voDT3h4XCIbDYLv98/Nkxe44cxGo1Qr9eRyWRgtb5erkGv+5vjbdYd0Gv/NtBrfznQ63550Gt/edBn7OVA7/nLg97zlwO95y8Pr7r2bxzoamhoaGhoaGhoaGhoaGhMI96YuqyhoaGhoaGhoaGhoaGhMY3Qga6GhoaGhoaGhoaGhobGtYIOdDU0NDQ0NDQ0NDQ0NDSuFXSgq6GhoaGhoaGhoaGhoXGtoANdDQ0NDQ0NDQ0NDQ0NjWsFHehqaGhoaGhoaGhoaGhoXCvoQFdDQ0NDQ0NDQ0NDQ0PjWkEHuhoaGhoaGhoaGhoaGhrXCjrQ1dDQ0NDQ0NDQ0NDQ0LhW0IGuhoaGhoaGhoaGhoaGxrWCDnQ1NDQ0NDQ0NDQ0NDQ0rhV0oKuhoaGhoaGhoaGhoaFxrfD/ADLqi/we0u49AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 1200x1200 with 10 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_mnist(pred[0: 10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "63746175-2ed3-4357-be13-5549db622f07",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7oAAABnCAYAAAAuTD4BAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABrJ0lEQVR4nO2913ccV5blvTMz0kd6nwmTcCRoRVKUl6q6qzVrpld3P888zN83DzMP3Wtm2lR3T1V/JZWkkkiJDiBIeCC9jYxIb78H1jmMBEESBEHC3d9aXJQByMSNiBv3nLPPPobRaDSCQCAQCAQCgUAgEAgEZwTjcX8AgUAgEAgEAoFAIBAIjhIR6AoEAoFAIBAIBAKB4EwhAl2BQCAQCAQCgUAgEJwpRKArEAgEAoFAIBAIBIIzhQh0BQKBQCAQCAQCgUBwphCBrkAgEAgEAoFAIBAIzhQi0BUIBAKBQCAQCAQCwZlCOuw3DodDZDIZuFwuGAyGo/xMZ57RaARN0xCPx2E0vlmuQaz74XmbdQfE2r8NYu2PB7Hux4dY++NDvGOPB3HPHx/inj8exD1/fBx47UeHZHd3dwRA/HqLX7u7u2LdT8m6i7UXa3+af4l1F2t/Hn+Jd+zpWXex9se39mLdj2fdxdq/n7U/tHTZ5XId9lsFf+YwayjW/e057BqKtX97xNofD2Ldjw+x9seHeMceD+KePz7EPX88iHv++HjdGh460BUl9rfnMGso1v3tOewairV/e8TaHw9i3Y8PsfbHh3jHHg/inj8+xD1/PIh7/vh43RoKMyqBQCAQCAQCgUAgEJwpDm1GJRAIBAKBQCAQnDWMRiNXiuj30WiE4XDI/ywQCE4+ItAVCAQCgUAgEAgAeL1eTExMwOl0wuPxIBAIAAC2t7exubmJbreLRqOBdrt9zJ9UIBC8DhHoCgQCgUAgEAgEAAKBAG7fvo1QKITZ2VlcvnwZBoMB//qv/4p/+7d/g6ZpyOVyItAVCE4BItAVCAQCwbFjMBhgMBhgtVohSdLYf+/1euh0OiwbFAjOKiSZNZlMMJvNMBgMGAwGGAwGLJ0dDAbH/THPJCaTCQaDAXa7HV6vF36/n38Bz9xdbTYbOp3O2B4lEAhOLuJJFQgEAsGxIkkSJEmCx+PBrVu3MDc3x4d9g8GA9fV1/PGPf0S5XD7ujyoQvDMsFgs8Hg9sNhvC4TAWFhYgyzIKhQLS6TTa7TYqlQoqlQoHvCL5czRYrVb4/X5YrVbMzc3h6tWriMVi8Hq9MJvNGAwGkGUZkUgENpsN1WoVBoNB9OoKBCccEegKBAKB4NgwGAyQJAkWiwWBQAC//vWv8dVXX8FkMsFiscBoNOI//uM/sLS0JAJdwZnGbDbD7/fD5XLh0qVL+E//6T8hFAphZWUFv/zyC2q1GjY3N9FoNNDr9cbMkQRvh8ViQSgUgtvtxszMDC5fvoxEIsEJhcFgAKfTiVAoBLPZDLvdftwfWSAQHIBjD3T1rnYk2TGbzbBYLGOzkUajETqdDvr9PkajEct4BAKB4Diw2Wyw2+0wGAzodDrodDp88BR708GgINfj8cDj8SAcDsPv98PtdsNoNHJFd+/7QCA4i0iSBFmW4fP54PV6+bkIBAKIRCKw2+1oNBqo1+vodDqo1+toNpviTPQW0L5CFV0KdgeDATqdDprNJjRNQ6fTQaVSQaPRQLPZRL/fP+ZPfvbQJzdp/9dDKh+TyYTRaIR+vz+W6KH/pm9zoWdCPBvnl2MNdI1GIyRJ4r4su90Os9mMRCKB6elpmM1m/tpms4nt7W2USiV0Oh0oioJOp3OMn14gEJxXDAYDkskkbt68CavVitXVVaytraHb7aLZbIq96QBIkgSz2QxZlvHZZ5/h1q1b8Pl8+OCDDxCNRtHv99FsNtHr9Y77owoE7wWXy4UrV64gmUwimUxidnYWfr8fXq8XyWQS7XYba2trWF1dRb1ex/LyMra3t9HtdqGqqjBHekMo0WYymRCNRvGb3/wGi4uLMBgMKBQKyOfz2NjYwKNHj9BoNFAqlfgMWiqVRPB0hBgMBng8HkxNTcHpdMLhcECW5bFeaEmS4PV64Xa70e12UalUxhI9AJDNZrG2toZWq4XRaMS/+v2+SE6cU4410NVnZ2w2Gzf6z87O4saNG7Barfy1qqpytaRer3NGUyAQCN43BoMBsVgMH3/8Mex2O4bDIfL5PFqtFhsnCV4NZe+dTicuXbqEr7/+Gi6XC7FYDD6fD+12G91ulyWaAsFZx2azIZlM4vLly4jH40gkEnC73YhEIrhw4QJ6vR6i0SjC4TCq1SparRZqtRparRaazeZxf/xTiclkgiRJ8Pv9uHnzJj7++GOkUincu3cPlUoFd+7cwe9+9zs+g9IvIRk/OsiIUJZlTE5OsqKBZOKE2WzGxMQEgsEgms0m0uk0FEUZ61e32WzI5XKscKBrJa7X+eVYAl26qW02G3w+H6xWK3w+H6LRKOx2O5LJJKLRKCwWC3+P0+nEzMwMrFYrVFWFzWaDpmlotVpQVVX0q7xnZFlGMBiE1Wodk2uqqgpVVYVRhuBMQy0WTqcTsizD7/cjEomg0Wig2+2iXq8f90c8kdDebzKZEAgEEA6HEQgEEI1GIcsy7HY7Z/AHgwEajQY0TUOz2RR7yXuGJJ3UUmQwGMYOjHa7HXa7HSaTia+d3kCMnoNut4t2u41Go8EVFZG42B9q0SJVCK01XQuTyQS73Q6fzweTyYTJyUmWMhsMBlSrVfR6PSGtPSAmkwlutxsOh4NNp6gVpVQqoVgsolarodvtviANF/fwwaC9A3juai1JEpxOJ6xWK/r9PtrtNvr9Pnw+HweybrcbPp9vrKJrMpngcDggSRJsNtuYUViv18NgMEA0GkUymYSmaTCZTDAajRiNRlAUBaqqot/vo16vs/rhPFxHo9EIWZZ57WRZhs1mG/ua4XDIUnCqfu9dG9r/B4MBXzOS+J/k9/N7D3TpgGg0GhEKhfDBBx/A7/djdnYWt27dgtvthizL3KNF9Pt93Lx5E81mE9VqFY8fP0a5XMbW1hb+9Kc/oVKpoNfrodvtnosb97hJJpP4u7/7O8RiMTZq6HQ6+PHHH3Hnzh0+2AgpleCs4nQ6kUgk4PV60el0YLfboSgK/vjHPwpZ20ugKq7VasWHH36I3/zmN/D5fLh8+TImJibYlGo0GnG7SqFQQCqVElXy9wgdRqm9yGq1wmg0otPpoNVqwWg0YnJyEslkEi6XC9euXcPs7Cyb9JjNZuTzedy/fx+lUgm7u7t49OgR6vU6BoOBCMJeQqfTQT6fh81mg9VqRbfbxXA45GSD0WhEOByGLMvodDqIRqP48ssvUSqV8MMPP2BzcxOVSgVra2tQFIUrWoL9sdvtmJ+fx9TUFJLJJJxOJ3q9HgqFAu7evYudnR0UCgW0223hvXBIqGJuNBr5vnY6nbhy5Qri8ThUVcX6+joURcH169fxt3/7t5icnOR3gT4OGAwGqNfraLVasFqtCAaDkCSJW4Z6vR4SiQRmZ2fR7/fhcrng9XrR7/exsrKC9fV11Go1LC0tYXd3l4O2s35dzWYzLly4gMXFRXg8Hly/fh1TU1NjX0NFw263i1qthkqlMrZPj0YjtNtt9Ho9NBoN7OzsoFKpoF6vs5rtpHIsgS69PGVZRjweRywWw9WrV/HZZ5/B4/G89HtpoymXy7DZbCgUCjAYDFhaWoKmaSc6o3DW8Pv9uHHjBm8ovV4PrVYL+XweS0tL/FAIBGcVq9XKWedYLIZer4dyuYyHDx+KsRMvQR84JRIJ3Lx5k9U8breb1200GqHb7UJRFBQKBdRqNREcvUfoPU2HVKvVCpPJhOFwiE6nA5PJBI/Hg4mJCfj9fty6dYvbjWRZhtVqxdbWFgaDAVKpFHq9HlZXV0Vl/jX0+31omgZFUV6ogOvlnbIsYzAYwOfzodPpIJfLoVQqod/vQ5Ik7O7uchVN7EMvR5IkBINBTE1NIRwOw2w2YzgcotFoIJ1OY2tri6vjYh0PB+35FOjSjOKpqSlcuHABpVKJFVCxWAyLi4uYmZnZ98/q9Xqc9LRYLAgGg5BlmXvUu90uPB4PgsEgRqMRq4a63S5sNhsMBgNKpRJ2dnY4gD4PCQxJkhAKhTA3N4dwOIwvv/wSly5dGvuaer2OUqmEdruNUqmETCYz5o8xHA7RbDbRbrdZRUvviZM+DeG9B7oWi4VlarOzs5ibm0MkEuHMzKugjZvGUJhMJjSbTXz00UcolUrI5/NIp9OcdT7JGYazhNVqhcPhgM1mQyQSweTkJEtEGo3GcX88geCd0Gq1UCwW+eAeDAa5JUOwP7IsIxaLweVyIRqNwuPxwOVysWSw3+9DURQ0m02kUimsra1hZ2cH6XQa3W73uD/+uYFMX8iJNplMwm63Q1VVPtRcuXIFly5dgtvtRigUgtVqZbUW8KxalkgkYLVa0W63sb29DZfLxdUCam0564fMN4EquoPBAA6HA2tra9A0DS6Xi2Wcejdyo9HIhm7T09MwGo3weDzQNA0ejweNRoPlzKKS/iJmsxnhcBjJZBKyLKPRaCCTyaBUKqHRaIxN+hC8HlIeUDWWWlSozc3v9/Oev7i4iEQiwcUtRVEwMzODwWDAfeekADGbzZAkCb1ej6+P0WhELpeDJElot9uoVqvodDpjLRfD4RBerxfAs/53+ruSySRGoxEajQYKhQLq9fqZVj8Mh0O0220oigJJklCpVHgONK1Xq9Xi+91gMLAyh9YeAO8j1CoRjUaRzWbRaDRgMBjQ6/XQbrdP3Dq+90DX5XLh+vXrmJ6exszMDH79619zb+7rDoj6jOb8/Dz6/T7PO2s2m7h79y5+//vfo1qtIpvNIpvNiuzxO8ZgMMDlciEQCGAwGODKlStQFAWVSgWdTgfFYvHE3fQCwdtCypKVlRUEAgFMTU1hYWEBgUAAPp9PjMJ5CeFwGB9//DGCwSCuXbuGiYkJ7hsCgHa7jfX1daRSKWxtbeHf/u3fsL6+jna7DU3TjvnTnx/sdjsmJyfh9/tx5coV/Jf/8l8QDoeRzWaxvb2N4XCIubk5zM3NcRXX4XBw8AUAPp8PH374ITqdDkKhEIbDIYrFIlZXV7G0tMQyOBF8PUfTNDx69AgWiwXFYhHAs2dmfn4eV69ehdPp5N5omlZhsVggSRK++OILtNtt7O7uIhgM8rV68OAB+5k0Gg3xPtZht9tx6dIlfPXVV6jX69ja2sLTp0+xsrKCcrnMSkGxZq+Hki5Go5ETMzabDVevXsWNGzfgcrkwPT2NRCLB/hbUo/v555+zz06n08Hu7i5SqRTW19fR7XbHVAz5fB6VSgXtdpvHPVHA2mq14PV6EYlE4HA48MUXX3Cyze12Y2pqCu12mw0Qc7kcvvnmG5amn1UZ82AwQLlcxvb2NhqNBra2tmC323mME/kvUI8uJczofO9yuWAymTjw7fV6UBQF7XYby8vLaDabMJvNUFUVxWLxxE1KeO+BriRJLFWLRqOIxWKIRCL7fu3LbjiTyQSXywUAcDgccLlcbB4QCAS48Zya0M/ijXsSoLWVJAkOhwOj0QherxfBYBAARGXrALwsINrvntV/7WECqZc9B+L5OBzUy0JBGvUC6U30BM8xGAxwOByIRCIIhULw+/182CEGgwFUVUU+n0c+n0cmk0E6nT7GT30+odYij8eDaDSKixcvcj86mb/MzMxgampqbNYl7SXD4RCSJCEQCAAAyuUyIpEIjEYjCoUCzGYzG5kInkOKBuBZUYDkg+RCTiO5hsMhJ/6p+mK32/nPSSQSMJlMaDQasNvt6HQ66Ha7oqViDyaTCV6vF7FYjPsMS6USz80VSZiDo293IDd9h8PBFXOPx8P90Pq+Wz3VahWpVAqNRgPlcpllyjRPejAYoFgssrR/d3f3hT7RUCiEbrcLl8sFRVH4nUzKB7PZjEgkAqfTCeBZDEGxwlmFEgiaprGhr6qq+84qJmgijt1u5zFPpNwcDofweDzo9/tQVRVerxdOpxPtdvul1/Y4ee+BLpXQyfVs74tuNBqhWq2iXC6zuRSZkNCm7nA4EI/HIcsyRqMRN6xPTU3hs88+Q61WQygUgsfjQavVQqFQYGMGkZ07GqgZHXh2zcLhMD8YVquVe7rEi3Uci8XCm4ZejgY8d7RrNps8KoLuV4vFAo/Hw98fCAQOFFD1+310u13+nVyB9X8fVeBpLI6QiB4MyigD4Fl+ghehlyk5nEYiEUQiEe7JBcAy1kajgVQqhadPnyKbzYr2k2OCDjhut5udlQ0GA5xOJ6LRKIbDIVwuF8vNy+UyVFVFp9NBtVpFvV5HMBjExYsX4fF44PV6sbi4iHg8zo7MmqZhd3dXKK9egqZpWF9fR6FQwGg04oO6LMtwOp0wGo0sFydjHqfTCafTiWQyCa/XC5PJBE3TUK1WsbOzI3qk/wxVssxmMzvz0v6jaRoajYZYpzeEzjQWiwXz8/P48MMP4fF4OCFGc3H3JunJi6HdbiOdTuPBgwdQFAWZTAZbW1vodrtwOBxwOBwYDodQVZVdyamiq48lqMe03W5zstTlcsFoNMLv93OQ1u12YTQa4fP5YLfb0ev1uKp71qB1y+VyqNfrsNlsSKVSAJ47Ye+HyWRCLBZDNBqF0+nE/Pw8JiYmWJ5OFd9EIsGJuGKxeOKMI997oEv6bkVRWH+/9/9nMhk8evQIzWYTlUqFM5x0QeLxOL788kt26SQZz7Vr13io+t27d3H37l0oioKffvoJzWaTLcjFgfTtURQFS0tLKBaLkCQJFy5c4L4Mh8MxNiZE8BybzcY96lNTU7h8+TIcDgc7V/d6PWSzWe79JCmJ2+3G3NwcfD4f4vE4rl+/Drfb/dq/Tz9UnQ6W9BKnisqTJ0/w8OFDNBoNKIoinpEDMBqNUK/Xkc1m2ZxBHIz2x2g0cr9PMBjEzMwMYrEY+yyMRiM2tFNVFSsrK/j+++9Rr9ehqupxf/xzCfXoBgIBuN1uzvp7PB6uhFCA0Ol0sLW1hY2NDSiKguXlZWSzWVy7dg0+nw8ejweRSIQPlzQ6pFKp4I9//CPy+bx4dvahUqng3r17kCQJhUIBpVKJgwWn0wmz2Qy3280J0xs3bsDpdLKr6mAwQCQSgdVqRaVSwXfffYednZ0TJys8DqgyToGuJEkYjUbQNA3FYhGapolq7htisVjg8/ngdDrx4Ycf4r//9//O9x/161JwpKfZbGJrawuVSgXLy8v47W9/i2w2i3q9jlqthsFgwImJ0Wg01ttP5yP67wC4t9pisWBjYwNra2vw+XxIJpOIxWIwm838eRwOB6LRKLa2trhH9SwGuoPBAKVSCdVqFSaTCaurqwcqlJjNZszPz2N+fp6LK4lEAkajkacjBAIBXLx4EbIsw2g0YnV19cR58xxbJLJXUkzVVjIwokwNXRzgeRbOYrFwRVj/EDmdTthsNnS7XYTDYYRCIa4iUC+AmO16NPT7fTSbTbZ6p2tJ8hXS/QvGMZvNLLf3+XwIhUJwuVx8b3a7XW74p8CXDBXC4TD8fj/i8Timpqbg9XrHZkfrjQWITqcDh8OBZrPJzrX6ER/9fh+FQoGrM81mU1ThDwiN1NorcRP3/Th6lYfD4WBJm8ViGavokqKAAtxWqyUOm+8Zuh7ktKy/TpTF1885pmumaRrK5TKq1SpyuRwymQyi0SharRYGgwFXe8gp2OfzYTgcjsnWBePQXk3zcSnQJZMei8XCqjeDwcDPC6neDAYDJysMBgPcbjcsFguPLDqv5yD95A9JksaUZzTzWYypfHNofyDlQTgcRjQafeHr6OxPQWqj0UCtVmMlZ6FQ4H7bw1TWh8Mh3+OtVgvNZpPltqSAsNvtLK+m/0YuwmeVXq/HSa6DBqKSJMHtdsPr9XJSk9C/EygBZ7VaT+QaHovrciwWw9zcHGKxGL/oVFVlp7t79+7hm2++4SZzknGS9jubzWI0GiEWiyGRSODGjRvweDwskTObzZiYmOA/12QyIRQKQVEUPH78GLlcTvTuHjFiLV8OJWckSUIsFsNHH32EWCyGeDyOhYUF2O32sUHcyWSSXQDJHMFmsyEQCMBut8Pj8fCcUXIYHw6HsNvtPO+S6Pf7HGjQZqWv2A4GA7adV1UV3333HX766ScOuEWg8XIoMUfJiL0jWehrzvuzEYlE8NlnnyESieDKlSvc6+l0OmEwGDAYDFAoFNg1P5fLsVOsuP/eHyaTCbIsw2azYXJyEtevX8fFixcRDofH+j+BZ/vG5uYmtra2oKoqfv75Zzx9+pSl57VaDbu7u3jy5AkAcK8vHYQpYNMnOwT7Q54j6+vrY1JlfXI/FAoBAIrFIo99kmUZbrcbFy9eRL1eh6ZpqFQqqNVqSKVSfI46b5CE1e/3I5FIsEKBHK93dna4dU5wdNDISUpmPn78GJlMhu9tSpIVi0Weifu29ye1a1ksFo4nrFYrqw8pAef3+2E0GqEoimjd2gMVU152lqFz6kkuIh5LoEsHfBpJAGBsaPRPP/2E3//+9y+1/Ha73SgUCvD5fLh16xa7qtGNa7FY2N2t2Wyy21sqleKMkejXPXrEWu4PSTetVismJibw+eefY25uDn6/n5M9+vt8P+c/+v/0i3p5VVVFpVLhqi8Fs/rvM5lMsNlscDgcCIVCYwGZwWBAu91Gq9ViifOTJ0+4Wi8CjZczHA45S6o3h6Fqgf56nWei0Sj+83/+z7h48SICgQASiQTPNKQez3w+j5WVFWSzWWQyGVQqlTG1guDdQ8Y8NOPy5s2b+OCDDziw0tPv97G+vo7/7//7/1CtVnHv3j2srq6OeQJ4PB4sLy+j3W4jmUzC7/dzoEvGJWaz+Zh+2tOFoijQNI2TAvTs0JknHA7DYDCgWCxiYmICbrcbDoeDryftU81mE+VyGcPhkMcYnTdMJhP8fj9mZ2cRj8fHAt1sNouNjQ2u6gqODkrOU6/ov//7v+POnTvQNA2pVAqapo2p2N72vUkV+nq9DkmSUK/XWc3mcDgAPK9YBgIBDIfDlxoznWeoAPOqQFf/NSeRY3FdttvtbJNPh3Ia+EwbeqPReKkRSavV4t4t6uGlvhUyPaHh1MCzbLLf70ez2YTX64Xb7Uav1xPSuLdgNBqNHfKB59IVq9UKm80mNo0/Q70gdrsdLpeL+6ooG68/7O2V83e7Xd5AyHCB5ILkUEtBQbfbRbfbfem6S5IEm83G14gk/waDgZ089Zb/4vq9GpKak0xKL+Uxm81jfUPnGcqaUxVXP2sVeD6IvlarsaGRWLf3DyXkaJwE7VnUikLoZZ568x6SGVKijuY2VioVBAIBdLtdVpzYbDYeKUhJD5IyCl7kZVLjfr/Ph3hFUVAulyHLMlqtFrrdLu/1JC90u93odru85ucRmnVO72Daq4fDIZsxitm5bw697yjZ1Ww20Wg0+PzS7/dZUVAul1EqlbhFkfaQo6bf77NyhMaZmUwmVgsNh0P2ldmrhjvP0FmGiiQ0Pm7vWZUk6CQzpzFNJ433HuharVbE43HMz8/zi240GiGfz+Onn35CPp/H1tbWKwNQkpjQ8GOn08kmJ2Tu4/V64fF4IEkSZ+1CoRA0TcP09DRyuRx++eUXlEolUXU5BM1mE7lcDq1WC9VqFYPBgB3azGYzcrkcvv3223P7MtXjcrlw8+ZNTE5OYn5+HtPT02zS8KpgstVqYXV1Ffl8HvV6HZlMBvV6Hc1mkyU2nU6HB3TbbLZX9kjYbDb4/X5YrVbMz8/j448/htvt5oHgVqsVgUAAk5OTUFWV+7AFL0LGJbu7u1BVFY1Gg18IPp8PwWCQXZnP+xpaLBb4/X6Ew+Gxe57kTs1mE5ubm/jxxx+5T0vw/rHb7bhw4QIWFxcxOTkJn8/HiTB9JZF+pwMOHSApwKV3abVaxY8//ojV1VXUajUkEgmEQiF2ZdU0DWtra0ilUry/kYu54GCQqqRWq+H+/fvY3t7G4uIiwuEw2u02j3KUJAkejwfT09OQZRlLS0tjAd55Ov+YzWZMTk7i5s2b8Pv9cLvdnEBut9tsXHoSD+wnGTK+rNfrWFlZwTfffAO/349qtcpTHarVKmq1GjRNw9LSErLZLPeZHzWj0QjFYhGPHj1inx4K2Hq9Hsv5g8EgFhYW2EhJ8Oydbbfb4XA4cPXqVfzlX/4lfD4fOy5T4aXf73Mstbq6ilwuh3a7fdwf/wXee6BrNpsRCoUwNTXF/200GqFSqeDRo0dIpVIolUqvzOj3ej1+IZJ5gMvlwu3bt/mgSZIEkvWEQiEEg0F0u11MTk7i8ePHWF9f50PVedroj4JOp8O9FLVajYdMh8NhBINBruAIwMPJr127xr25fr9/X/MoPZ1OB5ubm1heXkalUsHKygo7QpLb79779lV/nizLiMVicDqd+OKLLzAzMzPWl0d9vPF4HDabDcVi8cjW4CxC/gHNZhOtVgtWq5XHrgSDQWiaxomJ8wy5w/r9/rH/Tln+druNVCqFhw8fstOm4P1jtVqRTCZx48YNdkbWJ+L2Brt6RQOpGvT7Ua1Ww9LSElcTP/nkE0iSxM7xrVYL8/Pz2NzcRKVS4R5SwcGhYEzTNDx9+pRbUa5evcrzjgOBAM9FpjYvOhtR5fI8nX9MJhOi0SguXbrE6gV9JVKMNDscvV4PiqLAZDJhe3sbd+/ehSzLSKVS2N7eZoUHOVqTOzLwbs7fw+EQlUoFg8EAsiwjEokgkUjwmKJ2u41GowGv1wtJkqBpmmil+DM0xtLtdmNhYQGff/45z1UnXw3yhykWi3j8+DEePnzI5pwnjfce6Oqd7fSmLfoREwfJMNL/p8wMzd8tFovo9/twuVwIh8OQJIkdgM1mM7vdUsVXlmXO5IkM3sGhQ44kSWOmAfr+T6pw0UHoPMkRTSYTXC4X7HY7otEoAoEAfD4fDyynwyIdMvTW9iSfqlarSKfTPCC9VquhXq+zRORNs6C0sZOskKQ7dN/TM0h/9nm6Xodlvz5Skj8JNcMz9L3LtE/Q73T/6d1Oj+K+o/aVvcHZQTnpPUfvCv3M471rNhqNWLasn5LwsutF1UZ659MhiGT+FHwFAgEAEA7Mbwndq3T4pAQmqePIbVbfPkDzY88TdO9R6w71LutnsR4Wms1LPg37yWHpfEvvW3oPU7LotKJ/F9JoUPpd0zQOjGid34c8nM6pNAKwXC6j0+nA7XaPGUfSu+K8v7NpDWg2LrV9kscMVXO73S63GlWrVW6VOKmS//ce6FLD//r6OmdZqBJCPZ9vstm0Wi3kcjkOHjqdDrxeL/7iL/4C0WgUNpuNDRtsNhuSySQikQgkScLjx49htVpRKpWws7NzIkvuJxWSqTQajbF5yHSTm81mxONxXLp0CfV6HalUiuchn2X0G8Wnn36KS5cusevs1NQU96QBzw/5/X4fW1tb7Fq6sbGBVCqFRqOBdDqNSqXCG0un0zm0G22320WpVILZbEa5XObNia7dcDhEqVTC+vo6y3oEgqNAH3DuDXIpWVmpVI7EBIZ6gm02Gx843/QAQ9UJUk2c5gPoYXhZlU+/Z+kPrvvtR+TjMBgM+PpSoDUcDiFJEubm5mAwGJDJZLCxsYHV1dUTeVA6TeTzefz7v/873G43fvOb32BxcZF9IUwmE+x2O/x+P7eNUVLnvEAmXuQjUiqV2PH9bau5drsdoVAIDocDwWCQ3/mE3tuk1+uhWCyiVquh1Wohk8mc6neufsZtLpfDjz/+CEmSuFeXAiQK7N/HPUfeP+12G8vLy9A0DaFQCH6/HwsLC2PFtvOOfuTWtWvX8Ld/+7cIBoO4cOECGwlSkaRSqeDOnTs8P71YLHKx8CTu3+/9Cvf7fSiKgnw+j36/zz2D9PJ8U0MKqhZSgDEcDuHxeHDhwoUXqsYkmwaeSX0SiQSPZkmn0+/k5z2r0LXSj7fR3+BGoxFerxcTExM8A/A8QPehzWbDwsICvvjiC3i9XszPz7Mzpr6aS71BhUIBKysrqFaruHv3LpaWltgwjWb6ve0Gop/L2Gg0+BBK1244HKJer6NYLHLVWCB4V+jHFtDIiaPo1SJTJYfDwVWVNzVWo8PZaa+yvC179xz9GDQyxaNk2X5O8dSHTcZV9XqdkwfUVkRmMKLV5WhQVRWPHj2CJEmYnJxkKSG1pwDgqm6v1zt3Bjx6kx3gWS85yebfdqSQ2Wzmvt+pqSlcu3YNsizz/6e9js5N29vbKBQKbNB0mgNd4HmCTFGUE1HYoHNqt9tFOp3m1ph2u829uuft/n8Z9FxQD/sXX3yBSCQCl8vF9zAZ+Kqqis3NTTx8+BD5fB61Wu1Ej+N674EuPegk1XzV4d1gMLA7M/X5WK1WfmnqD0X6gehmsxmdTgeVSgWdTgeBQAAWi+WFP5ucH0m6IDgcrVaLK4UOhwMOhwMmkwk+nw+JRAJmsxnr6+vH/THfOUajER6PB263G8FgEOFwGH6/Hy6Xi+dFkjS43+/zy7XZbGJ1dRU7OztQVZU3jaOy2dd/PpJr0aFH77xMnMSMnECwH1arlQNZahWw2WwIh8OQZfmtAl2/38+mbPSC15vV0K/zBO1ZmqYhl8uxfO11h5xOp4NqtQqr1YpIJDKm/KFrJlzejwaz2QyPx8POwnsds/W/nydIkq832qFnm+7pN020UUuc1+uF3W7nCpjX60UkEkEwGOTKOSX2Wq0WS6cnJibg9XrZgZja96h4IHh7qG2GEmo0WcJsNr8wAeC8QcGt0WiELMsIhUKw2+2IRCKw2+18bqXEZjabRbFYZAVEqVSCoignfnrNsVV0C4UCZxRfhtlsxtTUFDsFzs3NIRKJoFqtYnl5+QV3TovFAovFAqvVilqthkePHsHlcuHy5ctwuVxjm7vFYoHH40EgEECxWDzXN/vbMBqNUCgUcO/ePWQyGczNzWF2dhY2mw0XL16EzWbD1tYWVldXsbW1ddwf951CUryrV68iGAzi9u3buHz5Ms+NpErqzs4O6vU61tbWcOfOHdRqNaTTaezs7HBli3pp96uUHBaLxYJAIMCBgM/ng8fj4c1MIHhX7O3NPQqMRiP8fj8ikQhkWcbly5eRTCbhdDoRj8fZZIRGar0JmqZha2sLiqJAVVVks1m0Wi2k02msrq6i1WqxEdl5SQwNh0NsbW3hu+++Q7VaxZ/+9Cesra2h3W6/Vv2hKAqb6fl8PvT7fVitVjgcDlgsFqiqKnp0jwhZlrGwsAC/34/p6Wmxrn+GAlyamxqNRqGqKkqlEjsAv4l0mXqcvV4vPv74Y0xPTyMej+P27dsIh8MAnu93FMCSjL/RaMBkMuHGjRtwOp3Y3t4GADx58gSKohyJjFrwPMgl1/FYLIZoNMoFCBo3dF4xmUxwOp2wWCyYmZnBp59+imAwiA8++ADBYBCyLLNxl6Io+Oabb/Djjz9CURQsLy8jm81y4fEkc2wV3WazycYUxN7DttFohNvtRjQahdfrxeLiIqamppDP56Fp2ktvUKPRyK7AVD3ei8lk4nmvYn7W4aEh4IVCAaPRCOFwGKPRiIeyj0YjdDqdsT6Vs4rRaITP58P09DSCwSCi0SiCweDYfU2jIBRFwfb2Nu7fv88jVYrF4jutEOlnWDscDu4X3s94RiA4ao46IDQYDLDb7ZywmZ2dxZUrV+B0Onk8DlUM3/Qwo2kaPB4PKpUK95bW63X0ej2k02k2WDpv1Go1bG1toVgssn/AQbL57XYb5XKZ2yPovU9VFVHRPTr047z2Omefl6TMflA1lwJep9PJ58NSqfTG8ksKomw2G2KxGObm5jA5OYlr164hGo2i0WhAURSW7uuN2IBnapTp6WkeuRWNRtlMVTwLR8N+82DJpI28e87z2Ycq3VarFT6fDzMzM4jFYojFYpyE7HQ6rGba3d3l6Qg7OzuoVqvH/SMciPce6FJGS1EU+Hw+PtibzWa2s6avs9vtSCaTuH79OlwuFyYnJxEMBiFJEq5evYp4PL7v32E0GhEIBBAIBDiDtxer1YpYLAaTyYRGo4FEIgGbzcaZeupFEvKRg/MuqjanDb0bKRmw0DisVquFfD6P5eVlKIoyNlZjv1FBR43D4cDMzAyCwSAmJiZgt9thNBpRr9f5s5LxFcmmBW8GHXxcLhcb7giOFr0ULRwO4+LFi/B4PJiamkIkEoHNZuMs9Wg0YnUEKSQAcG+pvo+Urh0pjUgBYbVaIUkS2u02z2FsNBrI5/PI5/PodDool8snoiftMEiSxJl9qpBTG5Aemh2dSqVQKBSgKMqB9yy9++lJ7uU6bdBB3mq1IhgMwul0YmJiAlevXkU0GsX09DRfR+o7r1QqUFVV7PN4bg71pmthsVgQj8cRDod5D5qbm4PNZsPu7i7y+TwqlQoymQwbSFJFt9VqodVqwWazoVAoIBKJQFVVhMNh3LhxA5ubm8jn82zuc56vz9tgMBjg9XoRDAbhcDhw/fp1LC4ucmxArWTkGaQoyrnYm2ikJI0Zm5+fh8/nQzKZxMzMDLssNxoNNknLZDKoVqvY2NjgdrvTtFbv/RRGzq8OhwNut5sXi9zqBoMBHzZcLhc++ugj/M3f/A1sNhv36Pb7fczPz790oalvgnpv7Xb7C1kbt9uNS5cuYWpqCmazGYVCAaVSCblcDqlUijc/sckcjvMa7FKg2+v1UC6XsbOzA0VR8ODBA2SzWeRyOTx48IAdZhuNBh+03/W95vf78fHHH2N2dhYzMzOc7S+VSlhZWUG5XMbu7i6bAp3Xa/g2UK9LMBjkYExwtJBhhsPhwIULF/AXf/EX8Hg8SCaTnLw0m80wmUw83oL64mlPr9VqqFarnKnudDowm808P9bhcCAej0OW5TE30VqthmKxiFarhSdPnmB1dRW1Wg137tyBqqqn7n2h96rw+XyYmprChQsXuOqlZzQaoVgs4v79+8jlcm80joUSfyaT6bXeHIKDQ32nXq8Xt27dwvT0NCYnJ/H555/zzHSn08kGQalUCqVSie/hoxipc5ohhSEd6g+yFqQkuXnzJm7fvs1tSlNTU0in0/jhhx+Qz+exs7ODR48e8VQKGr1Ce4nD4cDi4iISiQQCgQBu3LiBL7/8Ej/++COePHkCTdP4HCqelzfHaDRiYmICN2/ehM/nw+eff47bt29zIhoAVymfPHnCSYmzjtlsRiwWQzAYxOTkJL7++mvMzMywtJsKfoqioNls4vvvv8f3338PVVWxsrKC3d1d7tk9LRyLdJkO+JTdHQwGY7PNyGDEZrPB5/MhFou9YJqzX5X2TaBA2mKxwOfzcXW5Xq/DZrOxgzP1SIqN5uXo58Kd53XSu8iSyYWmaVAUBdlsFru7u8jlcscm+SBJG7UC0PPUbrdRrVY5U3ce54ceJbSPnXeji9ehV828yb5BPVcWiwVutxvhcBgejwc+n48PMPoRRtRLS1WbwWDATvBk/NJqtXhEHX0eCgL1I4rIbI8kj5qm7RsUnhb0PWxk1kJjaPZraeh0Otx68SbQnqg32DvPkkHgeTV27z8Tr3uf0pmJWlJob49Goyw/pFmh1EKkqioriGjCxXl4Z+9dW/2/UxB6ED8M6ssltUcikYDf70cgEIDX60WhUEClUmHPjfX1daiquq9xHe0lw+EQRqORvQWCwSCfd89zEuJtoYREMBhEMBhELBbDxMTEmMqKKuyaprGS86yjV535/X5MTk5iZmYGdrsdXq8XZrN5bKIKjV8lI8KT3o+7H8dS0S0Wi+j1epAkCT/88AM2Nzfxyy+/YHd3l6WT+uyXXlp2VC9Hk8kEh8MBs9mM6elpfPnll9A0jXslW60WNjY2sL29jW63i2q1yhK48/JyOCjtdhv5fB7D4ZDXiPoi6EVMFXa9fPCsMRgMkMlk8Msvv8Bms2FjYwMejwf1ep1lyqqqvvesIT03DocD0WgUk5OT8Hg8kCQJw+EQqqpid3eX+5TEvf12UILsKI3EzhrD4ZBNV4rF4oFlsAaDAaFQCMlkEl6vFwsLC4hEIuzODzyf8d1sNpHL5fDLL79wUEujcJrNJjRN49mOZEri8XjgcDjg8/lQKpW4zzEWi8Fut2M0GrEb5dTUFCRJYjOb0xq46RMzeiXUu0zSnNa1OirMZjMSiQTi8Tg7UXs8Hr4fKUFDrS8UlNJBnNoi/H4/J3lu3rzJvenkttzv93mU3NraGn744QdUKhWuypyn5DS1x8myzEmtwWAAVVV5pN6rqlQmkwkTExOYmJiAz+fDxYsXMTExAQBYXl7GgwcP2HeDHGlfNVu03++jUqkAeKZo7Ha73EZA5yYqAp2Xa3QU0PpZLBZEIhFcvHiR5cp79zT99AuqvJ91JEmCz+dDPB7nfUeWZVgsFhiNRk6KaZrGY68KhQLq9fqpNUh774Fup9PhHgbSxHu9XmxtbeHJkydoNptczaXDIjXnkw32USBJEkt6nE4nksnkmGlAvV7H7373O3z77beo1WpYW1vjF1Cv1xMbz58ZjUao1+vIZrNot9u4cuUKRqMRZ5LJ/IEOUvq5rWeNfr+Pzc1NpNNpDvRNJhMfXuhnf5+SD719vMvlwtTUFBYWFvgg2+/3US6Xsbq6yiMOzkNW811Csh5RGX855Na+tLTEUsqDBrqxWAxffPEFAoEArl27hsnJyTFDQf075unTp/jHf/xHbG1t8dzXvf4LdNin6qbRaEQ4HEY6nUYikcDk5CQ+/PBDBAIBOJ1OVkO43W7Mzc0hl8vhm2++OZXBG1UFSUVF+/R+P8t+VUfB4bBYLLh06RI++eQT+P1+3Lx5E8lkkiXe3W4XhUIBq6urPPaG7l2qilutVszPzyORSECWZUxPT8Pv93PPrslkQqvVQrVaRbPZxMOHD/HP//zPUBSFEz9n8T28H6TOcLvd8Hg87ETd7/dRrVaRyWTQ7XZfaTAnSRIWFhbw1Vdfwev14urVq5idnUU+n8e//Mu/8CSQlZUVHrnyqr7ffr+PQqGAarXKM10pwKXn8KSPbTmJmEwmHlk2OTmJmzdvwu/3w+fzvbB/UY9usVhEvV4/F+ttNpsRDoeRTCaRSCQQDAbh8Xj4/UeBLvlO5HI57O7unuo2h2ORLlPAWK/XUS6X0ev1oCgKz9Y1Go0wm838tXpJgV7yRC/ewwa/9H3kwAw8l7E4nU4Eg0H4/X52f6aeCQD8kjgvL4pX0ev10Gw2YbPZ0Ol0xqrvFGRR4AuAq/Vnkde9LN83BoOBX5q0+Vut1jEpYbvdRr1eP3UGAyeBvfuPXsJ/Vu/xo4CUMfS8vEnyy2q1wuVycS8tuWcSVLGt1Wqo1WoolUr8njlof6gkSSiXyyzxajQasNvtHFBTEstqtcLpdL40ODwN0MGaMvqn9ec4TZDCxuPxwOv1IhwOIx6Po9frcWBrMBh49jAZeZFapN/vw2azIRQKIRwOs+eJw+Fg1RS1z5DZIEnOFUV5L+aHJw2S6Ovv8dFoxL37VDXfD9rnHQ4H/H4/m7WR+zqNzFQUBZqmHUjeSUZY+kT4XjWcmIjw5tB1Jjd36lPfb4/WPyvnVYH1srMKnWv05/jTmrh/74EuPdyDwYArpTabjTXyFATRBvDzzz/zvL1EIgGfz8eBMPU1UE/DUUCHGLvdjsuXL8PhcKBer+Pq1at8YLp37x7y+Ty63e65kTu8jNFohFqtxm5s6XQa5XIZdrudJRIkG/H7/ZwVOknB4FnG5XLh2rVrSCQSuHr1KjweDwBAVVXkcjk0Gg2sra1hc3MTiqII6fIbYLFYOBCivn6SmenNRwQvYjAY4Ha72f1775zzV2GxWFiCaLVaX/i+druNjY0NLC0tIZ1Oc7vAmwTTzWYTa2tryOfzqFar3Os1NzfH83nPApIkIR6PY2FhgWcPv6x/ViRvjhY6kOtHnOidv0OhEI9K1O8npEYg6TIpDABwgiedTrPSan19HZqm4dGjR+z+fx4Tmvrxemaz+Y2+z2KxwOl0IhqNYnZ2FpIkYWtrC/fu3UOpVMKjR4+wtbWFTqdz4NYkek/opaI0jigYDKLVanGl8TyfMd8Uq9UKr9cLWZb5d3LhFzx7P66urqLRaHCrA43Ti0ajvH5zc3OIRCIol8vo9/tQVRVPnjxBKpUa8+U5DRxLoEvygF6vB1VV+XBIi0ebcLvdxi+//IJarQa3240rV64gkUjwhkUleI/Hc2Q3MUm5TCYTLl++jIsXL6LT6bDUenNzE61Wi3tfTnM5/6io1Wo8eiOVSqFcLsPlciEQCMDtdsPn88Hv98Pv90PTNJZmCd49LpcLt27dwgcffIB4PM6BrqZp3De8traGra0t7lk8LZvXcbM34KIMqN5FW6zl/pBKZnJy8o0CXXLUp1F0+wW6rVYLm5ubuHfvHqrVKhRFeeP9hgJdo9EIRVHgdDoRiURgsViwuLj4Rn/WSUaSJMRiMVy9epUdp98H5/250Le2ULWEWn6oF9zhcCAUCr2QeNgrtadeUxqNWCwWce/ePRQKBfY/oX67arV6bvclUjVREv6giTVyeKc9YHZ2Ft1uF3fv3sV3330HRVGwtrbG7RdvUvWipAW1y+kDXZKpZzKZw/7I5xKr1cpnT6/XC5fLBYfDISrjf6bdbmNtbQ27u7soFAqIx+Not9uIx+Pw+Xwc6Lrdbh7DZ7fbUS6X0W63kcvlTp0HybGmpV9mTESLR5s3PfylUond6CjQHQ6HPNtQz36SD3pB7N2IyOmZKsUkcaBsK1WOh8MhS41I0lyr1XizOq8Br14GSxIQWmN9r67NZkO32xVOtO8BOgBRJtrn80GWZa5EdbtdqKoKRVHYhOO83r+HRT9snSoq+nmtp+lFcBzQvkCjgA4KVc5fdnChZCrtR4e5BvprSH+WXl4IPHebP81JDepddLlcbNIDYOw9Se82Mu06qp/zNK7XUTEcDtFqtaCqKkwmEyqVChtI0dmFxmTRXv6qe34wGIw5gZOreLVaZenym6oaziL7uS/rnan3W19JkuBwOCDLMmw2GywWCwenqqpy4v5t35/6M+R5kC7Tz0atbQc5F9LeTsWy/e5lfeWeXPP3JopoLyM3/tO8h78p+mJio9FAuVzmpDG56ZP0GwA7uo9GI4TDYUSjUW6xoD3lpKvXTrT+ajAYoFAo8OiHTCYDp9PJASkNud8rXaYDvsViGXOso9FGezekRCKBxcVFuFwudqWlP582QY/Hw66en3/+OaamprCzs4M7d+6gXC6j0Wiwjfx5gzKP+sHrehMGi8WCcDiM6elpFAoF5PN51Ov1Y/7UZxdKzNjtdoTDYczPz+PSpUtjzrTFYhF37txBJpPB5ubmuTBhOEqox476+J1OJwCwDK1erwu1x2uwWCxwuVxot9u8V58WRqMRWq0WGo0Gz+k9jVBF98qVK2zUAzxPMtNhkCSv2WxW7BVHQLfbxdLSEmq1GlwuF9bX1xGPx2Gz2eD1enm0YiKR4OkQpF4gZ2w97XYbOzs7XMX97rvvsLu7y5MkjjpJcRrZr6WE+m7dbjfP09bf3waDAYFAABcvXoTP58Pk5CTcbjdGoxFUVcXOzg4nGI4KOt/u9R44S0iSxD3lVqsVHo8HNpvttd/X7XaRy+VQqVTY9HFv4YpMAql/fW8A3W63sb6+jlwuh/X1dWSzWdRqtVcah50l6Lw+GAyQy+Xwhz/8Affu3UMymUQul0MgEEA4HOY4yO124/r162i1WgiFQvjyyy9RLpdx9+5dbG9vo16vI5/Pn2hH5hP9FNEIilqtBgDY3Nwcy2xS1ZWyNoTBYOBDvb6But/vo9lsjvVQGAwGXL16Fe12G6FQCAAQDof5/1GmiRrajUYjrl27hng8DrfbjXQ6zZnA89pLoa9+UFZSPyvRbDbD6/UiGo2i2+2+UX+M4M2h+XEkG4/H40gmk9wTNhqNoCgKnjx5gp2dHZRKJXF4PQRWq5VncNvtdp7BSrNZRaD7cmhf0EsJTxv6uaSntRXDZDIhEAhgZmaG53oCzwJdOvhXq1WkUikOmsQ9/fb0ej1sb28jnU7D4XAgn88jHA5DlmUkEgm4XC4kEgkOAkjOTAqzvc9Lr9fjIHd1dRUPHz7E9vY2gPNdOdejP5fQmuhVTwBeUJYYDAZ4PB4kk0n4/X42/mo2m9xDS8qRo2KvmvA0JQAPitFo5B5ah8OBWCzGM9BfRbvdRq/XQ6PReEE9SDidTkxMTCASicDv979wTbvdLlKp1Nj5p9FojDnxn3Xofi2Xy6hUKjAYDNx7GwqFsLCwAKvVClmWEQqFODZKJpNotVpIpVKcGCB3ZhHovgV62Q49+FQqpxfuXjms3glVX9Gll/deIwZVVXk8jsPhYE0/zaPTbzoki3C73fD7/YhEIizzok3vvEKyEjJk0LsIknR5v546wdGyd2SI/rlpNBoYDAY8E22/50FwcOhepj2J7v92uy0C3degTySS4YteCnxYTCYTZFlGIBDg4OCoobFqhUIBhUIBzWbz1AYUVEEiqSTw/F3ZbDY50KWZiuKePhpozyBTS4vFwolgOniTrNxut8PhcECSJHi9XjZEo4oY9fZ6PB423iE5rrhez6BCBxU76L1He4XZbEa1Wh37HjrT0H5O32M0Gtkkde9kkMNC+yCdWU/rfrIXGhdKCkyaQx6JRLiqGw6HOdnwKui62e12vi703qWzTTAYZPNTh8PBsQApDuv1OorFIjKZzAuzjs/Kmr8J9HO3221UKhWeFe90OiHLMhcGSeFqsVjgcDgQiUQwPT0Nq9WKQqEwdv45aet4ogNdyvqThMPtdsNut3MmrdlsAsALLncGgwHdbvcFi3fatPQbksFgwNbWFur1Omw2Gx48eIC7d+9ClmXcvn0b169fZzkRvVQSiQTC4TDsdjtnNH766SekUqlTK2E7CkajERqNBorFIvr9PoLBIADwzMlgMAhFUURF9x1DM3NDoRB8Ph+/ZBqNBnZ2dqBpGtbX15HP59lg4LxkMt8F1PPSbDZ5ZFoul3sjB87ziL4Pkczrms0mNE17q33U4XDgwoULMBqNWFtbw/b29gsH2LdlMBhga2sL3333HUqlEit7ThvUo0v9hxToUoWwWq3i8ePH+O1vf4t8Po9SqXSiM/enCTpct9ttpNNplEolSJKEtbU1Vqq53W4+A1Gi+PLly7h8+TJkWcbFixcxNTUFq9WKZDIJn8+Hfr+PcDiMcrmMTqdzJEHYaYekxru7u2g2mygUCtA0DYPBgKtUdA1IQUjU63VkMhke1USJ+2AwiJmZGdTrdaTT6Re+7yDoizj6EV/dbhetVgvdbvdU7iuE0WhEJBLB1NQUZFnGpUuXuFro9XrZAVvvD/Aq+v0+KpUKT4cgOX6pVMLjx4+hKAquXbuG27dvIxAIcEJoMBigWq2iWq0il8vhhx9+wHfffcfnVZEMAiqVCu7duweLxYIHDx7g22+/hd1uxwcffIDr16/D5XJhYWGB45+vvvoKV69exdraGkajEb9ns9nsiVM4nehAFxif8UezE1VV5YPLy+QGb1KlqlQqXL4vlUrI5/M8Fmd6eprn6tLnoT4m4FkzdzgcRjqdPpXyu6NE359osVh4E6KRCbIss5xc8O6gbDPZ6pO5WrfbRblcRrlc5hc9ZUFP88v0JEBmPTSTWFVVYfD1GvRz0K1WK8sz3zaQIk8Ayt7TjPSjhBKca2trqFQqUBTlVD5DlEwmkx3geUKYTI1SqRQePnyIVCp1zJ/2bKE3x3xdkEQ9ujabjXva/X4/4vE4APC/O51OZDIZTlwcxfN0FqCEwnA4hNFo5IkZw+EQfr8f09PTGAwG+/aJdjod1Go1GAwGTsDpVSNGoxHFYvFQn0u/B5IBldFoPDOzXQ0GA1wuFzv63rp1C7dv3+ZZ6JRcowD/dZCJG1UNKVm0u7sLk8mEXC6HmZkZTE5OIhAIcBWSqvmVSgXFYhGbm5t4/Pgxt9qd5jU+KkjtADxXF1itVlaW+P1+xGIxVv/Mz8+j3+/DYrHg/v37PB62UCgc80/yIicyMqO+LYvFgmg0inA4DKvVCr/fD1mWUS6XMRwOUSgUuHJ7VAdKmmc2Go2wvr7OFuWj0YgNqsgJUZIk7ivwer0IBAIs+TqPA9npZVKr1SBJ0onL6px1SAZqt9uRSCRw6dIlxONxHt3SaDSwvr6OVCrF6oPz1Jdy1JjNZu7d15vh6V/A520P2I9Go4Ht7W3IsgyXy4VgMMgSPZPJBKvViqmpKXz44YdQVRWbm5soFAro9Xqo1+tjSUvqL19fX2eHSAqUSQY3HA7hcrkQDoehqiouX74Mp9OJZrMJRVHG5OUHuT57XVfpQKZ3jT4LyTuSTL6L1hKLxQKPx8P9prReVLWixJBgf/RGSqqqolAoYDAYjJ199PfjWe/xPAwU1Ox1UZdlGeFwGIqivGBgSt9HZl4UFOmfk8M++5Ikwefz8XxeGlM3Go2gaRoqlQpL2E8DFKwbjUYOihwOB2ZmZjA/Pw+3241oNMrmaqQeabfbKJVKL+zzvV5vzNWXAn9KUtC+bzKZ0Gq1xtaS9mn9tdHPoNYnmcQ7en/oeSmXy9jc3ES1WkUkEuGEGylsPR4PFhYWYLPZYLfbOelDz9lJ4MQFukajEYFAABMTE3C73fjss89w48YN2Gw2fklubW3BbrdjfX2dLwJlIt4Gkre0221IkgRFUfDTTz8hFovhv/23/wan08kHKqpAxGIx+P1+zMzMYGFhAR6PB9lsFplM5txVc0ajEWq1GnZ2dtBsNnHp0iUAL1r6C94NlPX3+/349NNP8dd//deQZRmxWAwAkM/n8dvf/hb3799HvV5HtVo9MRvRacNgMECWZUSjUU7AAc8VJmchG39U5PN5/P73v8fq6iouX76Mr776Cm63mw9FHo8Hv/rVr7CwsIByuYz/+I//wPLyMmq1GtbW1jigBZ7tMVtbW/jXf/1X2O12XLt2Ddvb23C73bh06RKmp6cBAPF4HJFIBMFgkJOjW1tbuH//Pmq1GkqlErLZ7IH2aJrLGI1G4Xa7OXgg1USn09n3gHwaeVc/g9Pp5CoLVVqGwyGq1SqKxSJ2d3eFE/8r0Bs9plIpGAwGRCIRXLt2Db1ej4Nc6hslmXO32xXv3z9Da9jpdNBoNFCr1WC1WhGPx5FIJDhBRjONKQiiXlByIafAkxJ1h00m2Gw2LCwsYHp6GlNTU4jFYnA6nRiNRkin01hfX2fTpdOAJEmc9L19+zb+7u/+DqFQCMFgEOFwmFtUSB1J16NareL+/ftjlcDRaMR7Q7fbhaZp0DSNlSbNZhM2m437fJPJJH7zm99gYmKCPXT2KizpmpJx6tt6QZxl9G0Vq6uryGazcLlcLP8PhUL49NNP4fP5MDU1hb/5m79Bo9HAN998g0wmg2w2i3q9Dk3TTsQan7hAFwDr930+H5LJJK5cucIBJkkdotEoqtUqOp3OG81gfB00Ggd4lmmiUTjUtE7D2YHnmX6y4PZ6veh0OiyDPm/opct2u30siDqP6/E+IfkTyU1osD31dgHgPqT19fV9v5847Yf19wU9/06nc+ylKqq541APIvUO0r5Asj2LxYJYLAav14tCoYCnT58in89jOBy+0LdF1Y7d3V1Ofno8HtTrdcTjcTbzcTgc/F6YnZ1FMBhkiRvwrO+OJIKvg+a2OxyOMYmdfu76aa/ovuv92WKxcEWfDAnpfUGHWJF0ezm0n5DpTqVSgdlsRqvV4n2Ggi4yFTvobNLzgn7GOVV0JUmCLMtwOBzweDzc5rP3+6iiuHd+K33tYZ4fGlsZjUYRDAZht9thNpvZNFJVVXS73RMRKBwEvadOMBjEpUuXkEgk+GysvxdJgkwjPwuFAu/N9P9LpRIrz2q1GhRF4fmtjUYDDocD09PTbBrrdrsxMTHBn2Vv0m6/iq7g5dAa1Wo11Go1drN2u92sciAJv9VqxWAwwMbGBlwuF8dmJ4UTF+jqm/P1fSl691iPx4Pr16/D7/djdXUVtVoNxWKRs25HtTEMBgMeG/T06VN4PB4EAgHcunULDoeDM3oA4Pf7MT8/D6/Xy6Y/gmfoHSFP6yiRkw4ZUAWDQUQiEbhcLpaw0Us4EAjgs88+QyAQ4O/b+6xQoqfVamFra+tE9lucFOiQRJJWkczZn2aziUwmg0ajgXg8jkqlMuYKDoD/XZZlTE9Po9PpYHd3FxsbGy/8eVRhGQwGSKfTfGCkSoHT6UQ8HofH44EkSQgEAnA6nZzNV1UVKysrAMCHqHq9PnbwIUk1fX8ikRjr+wLABmTUm3SaoZ+dfu90OiiVSshkMiiXy2/981ksFh53RiOM+v0+crkclpeXkc/nD2XmcxrweDwIhUKwWCw8loZ6Bo+6f/ZtAq/zQr/fRz6fx5MnT+D1erG4uAifz8f/3O12UalUkE6nWVpPaoN8Po/NzU2MRiP4/X7cvn0biqLA6XQim83uK4UdjUZ8rakFjkYuXr58GdeuXYPJZMLOzg4ymQyWl5ehKAq63e6p2ldozyTHXqre6hNbjUaD548/efIE2WyWDe/K5TL/WRRgkeqs0WiwqdpgMOD5x1NTU5icnEQymeR9hWII+mfB0UCTBiqVCmRZ5tZEioP0o7gsFgu2t7dRq9VORKLmREYc+iCXhnnT5mAymRAOh/H111+j3W7jxx9/RD6fh81mY7fTo1pYkh9Wq1X88Y9/xNbWFhYWFlgWR9WI0WiEiYkJfPrpp6hWqyiXy7h3796571GljBAZePV6Pfh8PhHoHjG00QSDQczPzyMcDiMQCLD6gTb7iYkJ/Nf/+l+hquq+2czhcAhVVXlsyj/8wz+gWCyKzOdLsNvtPEf3IMPuzyuqquLx48cwm80IBALsUOzz+bhCqu91pTnlS0tLuHPnDs8DJahXzmAwcELG5XLxeAQKRilQTSQSMBqNmJyc5Jnp3377LYxGI6rVKksE9fc5jXFxOByYmJjAxYsXuQ/JbDaj1+txkKyq6okcqfCm6D9/u93G7u4uVldXkclk3jo7b7PZEA6HWf5tMpnQ7Xaxvr6OP/zhD6hWq8jn86d+DfcjGo3i448/hizLyOVyPMqQfn+bn1l/oBeH+4PR6/WwsbEBm82GaDSKeDyOubk5xGIxfP7550gkEnjy5AlqtRparRYHu81mE2tra/j5558hyzLvJ4qiYHJyEru7u1z91StFRqMRisUiB8JkEhkKhfCrX/0Kt2/fRiqVwj/+4z/i6dOnyGazKBQKfG+clmeCxgfJsgyPx8PvRpJ2k2NyoVBANpvF//pf/ws//vgjr+1eRQf1U+slx3TWoUke165dwwcffIBIJAKPxzP2HAiOlsFgAEVRkMlk2BQPeD7OcjQaIRwO48aNG0gkEgCA7e3tE6HUOdERx94sM0EOg6PRCIFAgHtSjlLCrP8M1BdQLBYRDAbHAlh6sMhFjpz7xMP2HP1c14O66wkOBm36NNvM4/GwPFDfOzQajWC1WhGLxeDz+fZ9edJGpqoqDAYD7Hb7+/5xTg30wiVpOCVvTsuh5H1CBiJGoxGapqFer6PRaHA/GvB8H6U+rtFoxK0q9BKlBKZ+NihVxnq9HkqlEsrlMkwmE+r1OgdnekWQzWZDr9dDMBjk54AkyXqjGZI/U8WeqhMmk4mlj2SWQqY2ZwVaa6q8vO34Mb3pGF0H4FlijZxQa7XaiZK6HQW0/9rtdgQCAU7GkEP3Qc8re515KelPiSFx1ngzqLJYLpdhs9n4PEfmUCSlpb1n77zjSqWCwWCASCQCn88Hg8EAn8/HxlH7ue2TRJ/6gEnO63K5IMsye8Jks1keC3USKmEHge4/vUrHarWyooyg/uh6vQ5FUZBOp7G5uTkmJ37V36FvdbFYLKwSpGeL5kbvZ6hH/43k/PtJmwWvR2+Kt/f+pPe3LMtot9t8Bj0JnLhAl5rT19fXUSwWMTk5CbvdDpfLhdnZWYRCIf5a/QsAOLz73eswGo1sJe/xeMYcVoHnvak0ZqLZbIoHCOOH10AgALvdju3tbTFH9y2h3kCj0Yh4PI6ZmRnIsozFxUUsLi7C5XIhkUi8sNnTJkRjJ/aiN2dQVVVcp9fgdDoRi8UQDAbhcDg48DlLQc9RoA9Sd3d38f/+3/9DKBTCxx9/PCYFBp6P7TCZTJiYmOA2EVVVx6SEeythvV4P6XQag8EAPp8PnU4HT548gcfjweTkJB+E6J72eDz44osv0Gw2MT09ja2tLd7D6/U6vF4vLl26hFAohPn5eVai5HI5bG5uQtM0/Pzzz1hbW+NxUmdpz6deUEVRDj3VQH+wpEScLMtjxl1UFT9rPboOhwPBYBA2mw2Li4v44IMP4HK50Gq1sLy8DE3TXqv4kiQJdrud1TrkJhuPx7mnfWJi4gV35bN0H74LBoMBSqUSgGfKhfX1dYRCIfR6PQ5ejUYjCoUCz47OZrMYjUbIZDL4/vvvIcsy8vk8JiYm0Ol0kMvleFqHXm5MZ1JqyTAYDPD7/Tzf9enTp9jZ2UE2m8Xjx4+RTqfRaDROzbNAVVyakPLBBx8gEAhgZmaG21LI+KnZbGJzcxMPHz5EPp9HpVI5UL8sSWI9Hg9sNhump6cRjUYRiURw5coVTE5Ocr/63nWjoHgwGMBiscDn86HZbMLr9bLKsN1unyqJ+PuG1FayLGN2dhaLi4tIJBLwer0vfG2324WiKKhUKidqfveJC3RHo2fDn6vVKuv8u90uwuEwXC7XWKCrhzaUd5HdNJlMPK7C7/dzoKv/u1qtFiqVCsrl8qmyhH8fmM1mNoNZW1t7J3MtzxNUHbFYLLhw4QL+6q/+CoFAABcvXsTFixdZ9r9foOvxeF4waCD09+x+roWC59B8wImJCUQiEQ6+xEtzfygTTEEiqQ4+/vjjMdk37bVOpxP9fh9ffvklZmZmsLm5iR9//BHlcpmDBH3w1e12sbm5iZ2dHTidTmxvb8Pn8yEajeLGjRvcp0uGM8FgEIuLi+xim06nUa/XWaobjUbxq1/9CtPT0ywzlCQJmUwGv/vd71AsFvHw4UM8efIEnU7nzF1z/Rzder1+6ECXjJFojjrNztT37NHfcZYquk6nE8lkEj6fD9evX8cnn3wCp9OJzc1N1Ot11Gq118qWqa+Z3Hk//PBDuN1uTE5OsrtsIpEY82EgRLD7cqg3vFgsQlEUPH78mAOfubk5eL1euN1udLtdFAoFPH78GO12m71Xtre3YbPZsLGxgXg8DkmSYLFYOGGnHztEv2RZRiKRgMViYSdiVVXxzTff4PHjx6jVatje3oaiKCzTPQ1Qr7Esy5ibm8Mnn3yCRCKBiYkJ3tcHgwFXcp8+fYpvv/0WtVqNR2S9DhpXNDU1Ba/Xi88//xzXrl3jvzMQCKDZbKJcLr/Q8+5yudhJ22azIRAIoNPpwO/3w+Px8N591vbvo4Iq6E6nEz6fD5cvX2a3Zb/f/8LX06ioQqGAer1+YuKgE3mSpQyQyWTieWJk5NDr9cbkB2RprpeXmc1mvnkpW3TQjZ+Gdutf0i6Xizc/cljd+2LZO+tL8By9hn+/tRO8GSTvtNlsPL85EAiwFGqvJI4qtXtdB+n5AJ4HIuRoeJSzqc8qVFmXJAmdToflrGLdXg7NTCVTKE3TeM+lxIq+1YGc7DVN43YV+jraa/ceUgaDAbrdLlqtFprNJhqNBsvwyRWUHJ0pmCCpVbVaRbfbZfUOPU+NRgMAOJlZLpe5N/csejGQ3Jiuy2FdZUkWTkoSquaS6y1JQvV70VmAziV00HY4HHA4HJAkic83AHiUjb6/liq0sizD6/XyAT0QCMDtdnNFkJKdwPPzBwUV+t5GwYvQNWi321BVFeVyme9L4HkANxwOxyqwesNTmsVN6ir9L31LERkn6efH0n2vaRrK5TLq9fq+FcmTjiRJfD6mM7LH42Gz1tFoxJJvTdPYwfcgCg5aS6omkmGY3+9nhaDZbOaZu2QqCDyXKg+HQ74W+jM97W39fl+cR18DvYttNhsni2VZZofyvWdISvqfpHv5RAa6RL/fRyaTwWg0QqFQYFdj6nkhM4Ff//rXuHr1KrLZLHZ3d9FoNLC7u4vd3V12xqTG6VdBs+lILh2LxXjA9czMDDe875fJELyal/VbC96ceDyOr776CpFIBAsLC5zd9Pv9L8j3u90uSqUSux1S1lNRFOTzeT5k0ou/1WqxnHBjY0NcrwNA87fz+TwKhYJoXXgF1FM7GAywtLSEf/7nf4bX68X09DQSiQTMZjOcTifsdjtkWcbCwgJisRimpqYwMzODRqMBTdPYlTSbzSKXy40lF+hgpJ9zW61W0Wg0UK/X2Uyq3+/z7NGpqSn0+30Eg0FomsZZ7GaziWw2iwcPHqBarWJrawsPHz5Es9lEqVQ6s0kNp9OJ+fl5AMDGxgYymcwbz6oPhUL46KOPWGI4MzPDsvKdnR3eg2hvOitVFYPBAKfTibm5OXbrpiCXDob6PnAAfBCXZRkTExPsoL+wsACXy8UtKjabjYNnkuLTM7W9vY1KpYKlpSWk02lUKhV2JxfsT71ex507d5DNZhGLxaCqKiYmJmA0Gvn+n5iYwIULF9BsNlnOPBqNYLfbuQ+RAiin04loNMrOw4FAgBOhzWYT3W4X6XQa+Xye3d9zuRy63e6pSpjRzxsMBvGrX/0KFy9eRDgcxqVLl+D1erngNBwOsb6+jnv37qFareLevXvcJvK6/cTtdnNb0KeffopPPvkELpcLMzMziEajGI1GKJfLfO7/9ttvWWIOPAuUZ2dncfHiRTidTiQSCcRiMVbEORwOjEajE9NHehIxGo38fvb7/ZidncXs7Cy7awPPA9zBYIBarYadnR3s7u6iXC6fmL3nxAe6ZD9erVaxsbGBWCwGt9vNGeJQKITPPvsMnU4H2WwW29vbqNfr+Omnn8ZGPxwk0DUajQiFQrh69Srcbjc++OADXLp0CTabDX6/H263G0ajUUhvD8jLqukiCHg7otEo/vIv/5KTLySJ2s9xsNfroVwuo1QqoVKpYGNjA7VaDalUCo8fP0a9XufeUqr2knSKqliCVzMcDqFpGrLZLCcVBPtDSZVer4fV1VVYLBZ4vV58+umnbDBC/ZxOpxMzMzOsPrh16xYGgwGbmbTbbSwtLWFpaemF7LHevISqrpTJNxqNXHm32+1YWFjgw20ymQTwrBWlWCyi0WhgfX0df//3f4/NzU00m03UarUzXzGz2+1IJpOwWq3o9XqHeuf5/X589tlnuHDhAuLxOKampmC327Gzs4NUKoVKpYJSqYR6vX6q5oW+Ctp/qSdzbm4O8Xic5doGg4FNz/TVcqr0BYNBXL16lRPuN2/ehM/n44qW3kVfP99V0zSsra1ha2sLGxsbKBQKLIM9C+v6rmg2m3jw4AGWl5cxNTUFi8WCcrmMZDKJTz75BD6fDwsLC7h58yb3866vr3M1du9e7/V6WbIeCAQwPT0Ni8WCdDqNnZ0d1Go1/Pzzz/j+++/RbDaRTqdRrVZPlbsyqT2oBeTzzz/H559/DrvdDq/XO3YW6Xa72N7exh/+8AeUy2WsrKwglUrxM/Cqv4Nag7xeLz788EN8/fXXXBm3Wq3QNI2TBktLS/g//+f/YG1tDQDY7O3WrVusBqJ57VQltlgs6Ha7ItB9BRToTk5OIhQKYWpqClNTU/vuQ6TSSqfT2N3dFdLlN0FfCqfZZp1OB5FIhOcmUlbN5XLxyIpoNIrJyUm022243W6oqvrav0uSJExOTiIcDrM9OvWSkQRLP4SdAgJqtKe5X287MuA0Q0kFvSmPcLc7WvQy+Xq9jlKpNDZzml5ERqMR3W6XR3coioJisQhVVVGtVqFpGmeZu93uWGKC/g7B/uxNKnS7XZZ8i3V7NXpXX3KgJUkwHWLooEIBADnb6xMx5JIaDocPtOZ6aSG1oNCYG03TOAih/TybzaJWq3H1hZRBZ6k9haSFZBxC601VKjIiof+mP5Drkwn0/00mE0sKo9Eoyz7tdvuYjLFQKKBSqbCfxWk66L8KaqkiRYDL5YLdbmdjLjJH0zRt7HtIrhwIBBCJRBAKheD3+yHLMq8n7fn9fh+9Xo+fITJRy+VyKBQKLL9/nZOt4PkhnRzAy+UynE4nHA4H8vk8t6OQJwDNTu/1enyvk9MyPT/07/TnSZKEYrHIs6IVRUG9Xmd550kJBt4E/XOvPx/Tfayn3++PjWmi+5L2jL37h36MaDQaZVk0PQfNZhOapkFVVRSLRRQKBZRKJTSbTe7zpz+fpOGDwYCr8ZVK5cjnV5816DpYrVb4fD7uK3c6nS8Y35EBFTnKt9ttfl5Oyv5zogNd/SZULpfxhz/8ASsrK5ifn+cHy2AwwGazcf+P3+9Hv99HMpnEl19+iX6/f+A+KrKJp+HuZFCgn8moP+BS8N1ut7G8vIxvvvkGxWIRqVTqXB52h8MhFEXBzs4OH/z1vUf6A5Lg8KiqyrM/KRFkMpkwNzeHubk5WK1W7icvlUr405/+hPv370PTNORyOe4HqtVq/KLVH4r0wa5gHP2YD9rwSUK1traGarXKwZvg5QyHQxSLRXS7XTgcDgyHQ5RKJTZt8fl8cDqdiMfjLEN2uVycbLRarRgMBnA4HEgmkwe6V/V7NwW0w+EQ+XweW1tb6PV67DJcq9Wwvr7OVcednR3U6/V9xyqcZugAuLq6CpfLxQdLOsQ7nU44nU5+x1KQRdJb6lGnvi23240rV64gGo0ikUjgww8/RDgcBvBs3+r1erh//z5++9vfQlEUrK6unpk1pftSkiQEAgGWTZJKAQBXCfeeR2gvsVqt8Pv9vPaUuK/X6yiXy+h2u8jlcsjn82i328jlcuwUvrGxwRXyarUqAt0DQtLLSqWCP/3pT1haWkI0GsXjx4/hcrng9/sRiUS4CkhzuenM2el0UCwW2XW5VCrx3ka+Mul0Gtvb22i1WpyQOKjS8CSiD0qdTifPxt7rDzIajTihWa1Wed44zcGlZ8Pj8cBqtcLj8SAWi8Fut2N6epplx9PT0zy25pdffsGTJ0+gaRo2NjZQLpfZWGyvuWYqlUKn04HD4UAmk8HKygra7TbW1tY4cXkW9p6jxuv1IhQKwe1249NPP8Vf/dVfweVyYXJyks88FMjmcjn88MMPyOVy7KitaZoIdN8EOoRrmoalpSU8ffoUqqriq6++wuTkJFexKNNG2f/Z2dkXDu4HQV+xfV1Apm+AJylosVhks4Lzxmg04t41s9k8NlBaBLlHBx1w9APYqXoryzIcDgfLc1RVxZMnT/DTTz9xj+5pfbmeBF4W6NbrdeRyOSiKIrLFB4D6mlVV5bmLvV4PDocDiqIgEAjwSJ/BYMB9iST3JEdPr9f7xi/T0WgERVG4Xz2TyWBzcxPtdnusVWZ5eRnFYpErNCflpX2UDIdDqKqKbDaLVqvFIyNoJBDNIKa5mGT8oq/mmM1mduEMh8P46KOPcOHCBXi9XszOzsLtdkNRFHbi3Nrawi+//AJFUfhAdBbQSyJlWUY4HEY8Hh/7moWFBSwsLAB4dQuP/j05Go1Qq9V4osPa2honOql/ut1uI5/Po16vn8n79F1C61Wv17G2tgaDwcAjf6h94vLly9w/HYlEYLVaEQwG4fV60Wg0YLPZOMlAI4KKxSK30mUyGWxvb3Ni+TRfo71KDn2/5n70er0XzLbIoIuM2iKRCBwOB8LhMI9ITCaTWFxc5F5om82GVquFzc1NfPfdd9A0Dbu7u1xFJCMqghLQ5XKZk0XFYhGDwYBVD2clyXaUGAwGOBwORKNR+Hw+XLx4Ebdu3YLdbmcjSL2xqaIoePToETY3N7G1tcWO8ieJEx/oEno5JfWjkPsszasjAxO9OYB+lt9B2G8AO5XnSTZEroblchmbm5tQVZVfNtT7dZo3ssNChyAxZuXdQhlkWuvBYMAHU5K6UR8iOUOSnERs6m8HjUqhFy9VtMjBU9z3b45eNthqtWAymdBsNjlDT+YmoVBozP3xsEkzMjfMZDJotVrY3t7G7u4uP1fkEErtF2f5mRkMBiiXy9ja2kIgEEA0GkUwGATwbBZsv99n0zuXyzV2oJRlmd03w+EwfD4fS8mpWkOyRb0/QC6XG5MxnhXojELOx41Gg+eR22y2F84genVBo9EYk10C4IogjcPZ2dlBs9lEKpXi84aiKCzZPK/njqOGzjGkmCqVSkilUrDZbOzSTIo/qjKm02kuepRKJbRaLU6YkWxXP/3gNKO/z9vtNorFItLpNGw2G9xuN88qB5732iYSCTidTjYItFgsiMfjrFhwu938u95FOZ1OsxoCAGq1GnZ3d7ntodFo8NnmVeu6t1WGpONnbQ86CEajkSvplMSlKTckG4/FYpiZmWGHa70kneIhurdzuRz7wBx2FN275lQFutQbtb29jf/xP/4H3G43IpEIFhcX4fF42JnQbrezzTlJsA5qprHf4anf76NUKqFWq6HRaCCVSnHT9cOHD1GtVtmI5jxLIWguYrfb5V6K4XDIvV30Nad9oz9uSqUS7ty5w1J96lf3+/2Ynp7m4enkMkvVK9F3+/ZYrVaEQiE4HA4EAgFOLPR6Pd4jznOP/mGgg7yiKGNVQnJPJpMqt9sNq9WKyclJLCwsjM3ffRMGgwHW1tawvLyMVquFUqnExj2UxKSDLj0vZ/V6djod3L9/H8VikSXj0WgUBoMBiUQC8Xic9xia/1oulzEajRAMBsfafEha7vP5OEhuNBpQFAU///wz/umf/gn5fJ4dOffOQj7t0P3T6/VQqVSwtbUFi8WCQCCAeDz+0jNIu93GxsYGcrkcBxAkgd3Y2GC1yO7uLo+3ocCJenSHw+Gpcu096dAB3mQyoVgsYm1tjccF2Ww2DgpoRI2+yEFnQPKW2Tvy6bRDgSglAe7cuYNms8mtCoFAgL+WnI+//vprNJtNVCoV1Go12O12XLx4EbFYjF3D6XeSOZfLZTx69IgrsKSW2tjYQDab5a+nYPVVZxv6rJSko89Pgd55wmq1Ynp6GqFQiOfetlotOBwObpv46KOP8PXXX8Pn8yEWi3GijvanWq2G+/fvI5VKYWNjA3fv3uVk8Unch05NoAuAA0iaxQUAk5OTGA6H8Pl86PV6nGWmHgpqqH4bySxVHBRFgaIo/KLe2dnBgwcPuE+XArvzDJk30MYFiNFCRw0ddOieDofDnKXzeDyQJIkNpqiiK+S0R4MkSXA4HBzgkgEHGRidpTEp7wvKsO+VngHPx65QJcBsNmNxcZGlb4dhMBhgZWUFv/zyC1qtFur1+rl1yqYkQ61W43dcr9eDJElwu90sE3c6neh0Otx/OBwOEY/HEYvFuEd37/WgUVCNRgO5XA6PHj1CNptFvV4/k+9KSsYbjUautlYqFe4n18/L3fs91WoVmUyGD+zD4RDpdBr3799nQ7Td3d0TeYg8i5DRIwBef8FzKEgk52jyy7ly5crY1xmNRvh8PszOzvIzoWkanE4nrl27hsnJSTSbTXa4p+kQrVYLqqqiUqmg0+kgk8kgk8mg3+9zwUmfFHod1CsszkHg/uhIJMJKEoPBALfbzcn76elpXLlyhV20SbJM151aJTY3N7G9vY1sNotCoXDMP9nLOVWB7n60Wi2exUfZNBoH5Pf7eXA7Bb3k3qYPxvTSIgrUKDtLwUImk0G1WuUeDHJ8owrmWcnWHRXU+5XP5/mQSpUamkXc6XTObT/zUSFJEhuXkMRnNBqh3W7zIfO8ZSzfJeQF4PF42OGaMtx0gBfrfXSQAoRaRigzv76+fugxb8PhkHtS9e7w55V+v49ut8vv0p2dHTYCI8ka9WeRQmc4HMLr9bI/BpnQ6Nt8CoUCHj58iHK5zN4aJO0/y+9K6ltbWlpCpVKB3+/H2traS+/XZrOJtbU15PP5scM7mexQIka8JwUnDRrrSXLmUCiEbDbLFW8aMUTmsTQ7l6TEFMwuLS2hWCyiVqvx3kzSf1JI0Hn9LPQ5v2+MRiNkWWaDu0uXLmF+fh7tdhsTExNoNpuQZRnBYJBHy5FygWKgXq/Hru7FYhErKyvY3NxkY7yTzKkPdGu1GpaXl1nyRq6HsizzUHWr1crObleuXEEsFoOmaUilUmg0GnC73QiHw5AkCfV6HaqqotvtolQqcdN6pVKBqqrcc6SvXJ6V3oujpNfrIZPJYHl5GR6PBzMzM2xaEIvFkEwmUalUkM1muTdJ8GaQ43ggEEAgEOAxFiQtyeVyLKcXHA00u9vv93NrBEn2yf1UVHSPDjrck9TMYDBwT9zbzD8klcN5nzNKgSkdJp88eQK73Y5oNMoHI3qnjkYj+Hw+NljSz4GlQLff7/MoppWVFfzDP/wD1tbWUKlUuK/0LL8r6V5Kp9P43//7f8NisfCvl92vJEGmai0ld/QeDGdppJXg9EP3qKZpePjwIZ48eYKVlRXs7OywpwK1mly4cIErvZqmoVQq8X4zHA6xsbGBv//7v8fTp0/ZV4QkxeTSTOdtvQnSWd1D3gWSJCEej2N6ehrRaBR//dd/jZs3b7JyodvtwmazweVyjTlpGwwGblnRNA0//fQTVlZWUKlUcP/+fWQyGR4Zd5I59YEuPQB6DAYDz0Kjvgqz2Qy/3w+32w2j0QhFUbC9vQ1VVXkkkcVi4R4ksvEnm3iSdpG0SDxkr4YcmBVFgcFg4MO/yWRi6Wez2RQuzG+JJEns9KmXl/R6vbH+LcHRQKZfVqsVJpNpLBAjubjg6NEHpFQ9FxwNtK70niuVSrBarbx3GAwGNpjZ6was/53ejdTGQ20+GxsbnFg4L8GakEkKzgM0kg149vw7nU5Uq1XIsszqSVKGkFOz3siq0WigWq1id3cX6+vrAMbHG4qk8dFATsoej4f7bqempvg6UPxDMRPwfOwWFfQ0TeNRfIqi8Jis08CpD3T3gzJArVaLx66YTCb0+32srKyw5LhYLKLdbnPjuyRJaDQaLJGgLAaZCQhXw4PT6/WQSqXw4MEDRKNRTExMIBqNjkk/6/X6W1VlBPtDPY/lcpkVCYKjodFoYGtrC6VSCYPBgBM59+/fF5Jlwamm0+nwPGGSI0ajUYRCIczPz7P3BakYVFWFpmno9XpQVZVdUHd2dlCpVLh3i6qV4t0pEJw96LlutVooFApQVRU2mw0Oh4ONMTOZDAwGA7LZLIrFIoxGI7cUkvKMkmD6xJngaBgOh6jVashkMjCZTGNGi9R6QqaxNFO6UCig1WphfX0d6+vrqNfrePLkCbu/n6ZE3pkMdAGwHAt4PvdLURSUSiWuxJDkmF7elG0iWYRekkwZJsHB6HQ6WFtbQ6PRwMzMDG7cuMGDwj0eD4LBIFRVfWHAuODtoU1N7wQuOBoURcHy8jKMRiMePnyIf/mXfwHwTJIlEgqC00yr1eK+K7fbjY2NDXi9Xty4cYNNScxmM/fokiFjs9nE5uYmstks96aSTJmMGoXbvkBwNqHnmpJcNKKTfv/ll1+4P51GdALgsZ96vxyxR7wbaIwctQlSG6bRaITVauUgl6ro6XQad+/eRbVaxQ8//IA7d+6g0+mwb9Fpc6s+s4Huy16s4jD6ftBLl1VVZUMNYf7y7iHzHkr2iJfH0UH9dADOrVOv4GxCzuHUR1soFHhOZq1W4xmZFosFg8EAqqpCURR2Sy0UClAUBfl8HoVCgQ3ERIJYIDj7iBFXJxe9yrXZbPLoSVK6UvGPvABoJGWlUkGxWEQulztVge1ezmygKzhe6CBEGbz/+T//J3788UeUSiWsrKygWq2iUqmIjVEgEAhOCKRgIhkizRfu9XpsvkbS5Uqlgkqlgl6vh3K5DEVRuIpLAa4IcgUCgeB4oUB3NBohn8/jn/7pn/D06VOYTCZ21tcbfdHc7maziVQqder3cRHoCt4JNF6IXPY2Nzc5a0QP3GAwENVdgUAgOCHQgabVaqHdbnNf3b17917wU9AHsuRfoW/5EQgEAsHJgBR+nU4H//f//l82GNwPOpuTOvC0n9NFoCt4Z+h7m0+z7OEkQ73mNF+OnH/b7Tba7baQigsEgjdG3/pDZowCgUAgOL1QIvK8ucKLQFcgOKVQH3SxWESv10M6ncbOzg475T1+/Jj7owUCgUAgEAgEgvOECHQFglNMp9PhPrpisYhCoYBGo4Ht7W1sbW2x+YBAIBAIBAKBQHCeEIGuQHCK6fV67P5bKBSwtbXFbtftdput4AUCgUAgEAgEgvOECHQFglPKaDRCrVbDxsYGzGYz8vk87ty5g36/j2w2i3K5jMFgIPqjBQKBQCAQCATnDhHoCgSnGL2pQDabPeZPIxAIBAKBQCAQnAyMr/+S/SFHRsHhOcwainV/ew67hmLt3x6x9seDWPfjQ6z98SHesceDuOePD3HPHw/inj8+XreGhw50NU077LcK/sxh1lCs+9tz2DUUa//2iLU/HsS6Hx9i7Y8P8Y49HsQ9f3yIe/54EPf88fG6NTSMDplOGA6HyGQycLlcMBgMh/pw55XRaARN0xCPx2E0vlmuQaz74XmbdQfE2r8NYu2PB7Hux4dY++NDvGOPB3HPHx/inj8exD1/fBx07Q8d6AoEAoFAIBAIBAKBQHASObR0WSAQCAQCgUAgEAgEgpOICHQFAoFAIBAIBAKBQHCmEIGuQCAQCAQCgUAgEAjOFCLQFQgEAoFAIBAIBALBmUIEugKBQCAQCAQCgUAgOFOIQFcgEAgEAoFAIBAIBGcKEegKBAKBQCAQCAQCgeBMIQJdgUAgEAgEAoFAIBCcKUSgKxAIBAKBQCAQCASCM4UIdAUCgUAgEAgEAoFAcKYQga5AIBAIBAKBQCAQCM4UItAVCAQCgUAgEAgEAsGZ4v8HFfmZw6T20QcAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 1200x1200 with 10 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_mnist(test_set[0: 10][0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca8c4d07-ea44-4d2d-b2a8-946a2d332cda",
   "metadata": {},
   "source": [
    "### GPT model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f176ac2c-28a2-4e4e-b00d-e1a775a46933",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\ntaken from: https://github.com/karpathy/minGPT/\\nGPT model:\\n- the initial stem consists of a combination of token encoding and a positional encoding\\n- the meat of it is a uniform sequence of Transformer blocks\\n    - each Transformer is a sequential combination of a 1-hidden-layer MLP block and a self-attention block\\n    - all blocks feed into a central residual pathway similar to resnets\\n- the final decoder is a linear projection into a vanilla Softmax classifier\\n'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "taken from: https://github.com/karpathy/minGPT/\n",
    "GPT model:\n",
    "- the initial stem consists of a combination of token encoding and a positional encoding\n",
    "- the meat of it is a uniform sequence of Transformer blocks\n",
    "    - each Transformer is a sequential combination of a 1-hidden-layer MLP block and a self-attention block\n",
    "    - all blocks feed into a central residual pathway similar to resnets\n",
    "- the final decoder is a linear projection into a vanilla Softmax classifier\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f16bd3f3-8e8d-4db2-b852-88f3f970b5a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "18e43eb0-2a5b-4de9-b8de-42f4d8242ece",
   "metadata": {},
   "outputs": [],
   "source": [
    "class GPTConfig:\n",
    "    \"\"\" base GPT config, params common to all GPT versions \"\"\"\n",
    "    embd_pdrop = 0.1\n",
    "    resid_pdrop = 0.1\n",
    "    attn_pdrop = 0.1\n",
    "    # 20, 16\n",
    "    def __init__(self, vocab_size, block_size, **kwargs):\n",
    "        self.vocab_size = vocab_size\n",
    "        self.block_size = block_size\n",
    "        for k, v in kwargs.items():\n",
    "            # 设置属性值\n",
    "            setattr(self, k, v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "bdf6e15c-5bda-49bc-9386-05d971f4fa52",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CausalSelfAttention(nn.Module):\n",
    "    \"\"\"\n",
    "    A vanilla multi-head masked self-attention layer with a projection at the end.\n",
    "    It is possible to use torch.nn.MultiheadAttention here but I am including an\n",
    "    explicit implementation here to show that there is nothing too scary here.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, config):\n",
    "        super().__init__()\n",
    "        assert config.n_embd % config.n_head == 0\n",
    "        # key, query, value projections for all heads\n",
    "        self.key = nn.Linear(config.n_embd, config.n_embd)\n",
    "        self.query = nn.Linear(config.n_embd, config.n_embd)\n",
    "        self.value = nn.Linear(config.n_embd, config.n_embd)\n",
    "        # regularization\n",
    "        self.attn_drop = nn.Dropout(config.attn_pdrop)\n",
    "        self.resid_drop = nn.Dropout(config.resid_pdrop)\n",
    "        # output projection\n",
    "        self.proj = nn.Linear(config.n_embd, config.n_embd)\n",
    "        # causal mask to ensure that attention is only applied to the left in the input sequence\n",
    "        mask = torch.tril(torch.ones(config.block_size,\n",
    "                                     config.block_size))\n",
    "        if hasattr(config, \"n_unmasked\"):\n",
    "            mask[:config.n_unmasked, :config.n_unmasked] = 1\n",
    "        self.register_buffer(\"mask\", mask.view(1, 1, config.block_size, config.block_size))\n",
    "        self.n_head = config.n_head\n",
    "\n",
    "    def forward(self, x, layer_past=None):\n",
    "        B, T, C = x.size()\n",
    "\n",
    "        # calculate query, key, values for all heads in batch and move head forward to be the batch dim\n",
    "        k = self.key(x).view(B, T, self.n_head, C // self.n_head).transpose(1, 2)  # (B, nh, T, hs)\n",
    "        q = self.query(x).view(B, T, self.n_head, C // self.n_head).transpose(1, 2)  # (B, nh, T, hs)\n",
    "        v = self.value(x).view(B, T, self.n_head, C // self.n_head).transpose(1, 2)  # (B, nh, T, hs)\n",
    "\n",
    "        present = torch.stack((k, v))\n",
    "        if layer_past is not None:\n",
    "            past_key, past_value = layer_past\n",
    "            k = torch.cat((past_key, k), dim=-2)\n",
    "            v = torch.cat((past_value, v), dim=-2)\n",
    "\n",
    "        # causal self-attention; Self-attend: (B, nh, T, hs) x (B, nh, hs, T) -> (B, nh, T, T)\n",
    "        att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(k.size(-1)))\n",
    "        if layer_past is None:\n",
    "            att = att.masked_fill(self.mask[:, :, :T, :T] == 0, float('-inf'))\n",
    "\n",
    "        att = F.softmax(att, dim=-1)\n",
    "        att = self.attn_drop(att)\n",
    "        y = att @ v  # (B, nh, T, T) x (B, nh, T, hs) -> (B, nh, T, hs)\n",
    "        y = y.transpose(1, 2).contiguous().view(B, T, C)  # re-assemble all head outputs side by side\n",
    "\n",
    "        # output projection\n",
    "        y = self.resid_drop(self.proj(y))\n",
    "        return y, present  # TODO: check that this does not break anything"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "03cfa89c-15ba-4755-87d7-18d0b94f5e4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Block(nn.Module):\n",
    "    \"\"\" an unassuming Transformer block \"\"\"\n",
    "\n",
    "    def __init__(self, config):\n",
    "        super().__init__()\n",
    "        self.ln1 = nn.LayerNorm(config.n_embd)\n",
    "        self.ln2 = nn.LayerNorm(config.n_embd)\n",
    "        self.attn = CausalSelfAttention(config)\n",
    "        self.mlp = nn.Sequential(\n",
    "            nn.Linear(config.n_embd, 4 * config.n_embd),\n",
    "            nn.GELU(),  # nice\n",
    "            nn.Linear(4 * config.n_embd, config.n_embd),\n",
    "            nn.Dropout(config.resid_pdrop),\n",
    "        )\n",
    "\n",
    "    def forward(self, x, layer_past=None, return_present=False):\n",
    "        # TODO: check that training still works\n",
    "        if return_present:\n",
    "            assert not self.training\n",
    "        # layer past: tuple of length two with B, nh, T, hs\n",
    "        attn, present = self.attn(self.ln1(x), layer_past=layer_past)\n",
    "\n",
    "        x = x + attn\n",
    "        x = x + self.mlp(self.ln2(x))\n",
    "        if layer_past is not None or return_present:\n",
    "            return x, present\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "24848400-ff1f-4bb7-a37b-e76db482e7eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "class GPT(nn.Module):\n",
    "    \"\"\"  the full GPT language model, with a context size of block_size \"\"\"\n",
    "    # 20, \n",
    "    def __init__(self, vocab_size, block_size, n_layer=12, n_head=8, n_embd=256,\n",
    "                 embd_pdrop=0., resid_pdrop=0., attn_pdrop=0., n_unmasked=0):\n",
    "        super().__init__()\n",
    "        config = GPTConfig(vocab_size=vocab_size, block_size=block_size,\n",
    "                           embd_pdrop=embd_pdrop, resid_pdrop=resid_pdrop, attn_pdrop=attn_pdrop,\n",
    "                           n_layer=n_layer, n_head=n_head, n_embd=n_embd,\n",
    "                           n_unmasked=n_unmasked)\n",
    "        # input embedding stem\n",
    "        self.tok_emb = nn.Embedding(config.vocab_size, config.n_embd)\n",
    "        self.pos_emb = nn.Parameter(torch.zeros(1, config.block_size, config.n_embd))\n",
    "        self.drop = nn.Dropout(config.embd_pdrop)\n",
    "        # transformer\n",
    "        self.blocks = nn.Sequential(*[Block(config) for _ in range(config.n_layer)])\n",
    "        # decoder head\n",
    "        self.ln_f = nn.LayerNorm(config.n_embd)\n",
    "        self.head = nn.Linear(config.n_embd, config.vocab_size, bias=False)\n",
    "\n",
    "        self.block_size = config.block_size\n",
    "        self.apply(self._init_weights)\n",
    "        self.config = config\n",
    "\n",
    "    def get_block_size(self):\n",
    "        return self.block_size\n",
    "\n",
    "    def _init_weights(self, module):\n",
    "        if isinstance(module, (nn.Linear, nn.Embedding)):\n",
    "            module.weight.data.normal_(mean=0.0, std=0.02)\n",
    "            if isinstance(module, nn.Linear) and module.bias is not None:\n",
    "                module.bias.data.zero_()\n",
    "        elif isinstance(module, nn.LayerNorm):\n",
    "            module.bias.data.zero_()\n",
    "            module.weight.data.fill_(1.0)\n",
    "\n",
    "    def forward(self, idx, embeddings=None):\n",
    "        # idx, (B, 49)>> (B, 49, 16)\n",
    "        token_embeddings = self.tok_emb(idx)  # each index maps to a (learnable) vector\n",
    "        if embeddings is not None:  # prepend explicit embeddings\n",
    "            token_embeddings = torch.cat((embeddings, token_embeddings), dim=1)\n",
    "        # t= 49\n",
    "        t = token_embeddings.shape[1]\n",
    "        # 49<= 49\n",
    "        assert t <= self.block_size, \"Cannot forward, model block size is exhausted.\"\n",
    "        # (B, 49, 16)\n",
    "        position_embeddings = self.pos_emb[:, :t, :]  # each position maps to a (learnable) vector\n",
    "        x = self.drop(token_embeddings + position_embeddings)\n",
    "        x = self.blocks(x)\n",
    "        x = self.ln_f(x)\n",
    "        # (B, 49, 20)\n",
    "        logits = self.head(x)\n",
    "        return logits, None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "e1d94fc9-1ab0-4972-bdc1-3255d6fe5834",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Transformer(nn.Module):\n",
    "    def __init__(self, vqgan):\n",
    "        super(Transformer, self).__init__()\n",
    "        self.sos_token= 0\n",
    "        self.vqgan= vqgan\n",
    "        # codebook vector num, block_size , layer num, head, embed num\n",
    "        self.transformer= GPT(30, 49, 2, 4, 16)\n",
    "        self.pkeep= 0.5\n",
    "\n",
    "    def encode_to_z(self, x):\n",
    "        # (B, 16, 7, 7), (49B,),\n",
    "        quant_z, indices, _= self.vqgan.encode(x)\n",
    "        # (B, 49)\n",
    "        indices= indices.view(quant_z.shape[0], -1)\n",
    "        return quant_z, indices\n",
    "\n",
    "    def z_to_image(self, indices):\n",
    "        # (B, 49, 16)>> (B, 7, 7, 16)\n",
    "        ix_to_vec= self.vqgan.codebook.embedding(indices).reshape(indices.shape[0], 7, 7, 16)\n",
    "        # (B, 16, 7, 7)\n",
    "        ix_to_vec= ix_to_vec.permute(0, 3, 1, 2)\n",
    "        # (B, 1, 28, 28)\n",
    "        image= self.vqgan.decode(ix_to_vec)\n",
    "        # (28, 28, 1)\n",
    "        return (image[0]).permute(1, 2, 0)\n",
    "\n",
    "    def forward(self, x):\n",
    "        # x, (B, 1, 28, 28); indices, (B, 49)\n",
    "        _, indices= self.encode_to_z(x)\n",
    "        # (B, 1)\n",
    "        sos_tokens= torch.ones(x.shape[0], 1)* self.sos_token\n",
    "        sos_tokens= sos_tokens.long().to('cuda')\n",
    "        # mask, (B, 49)\n",
    "        mask= torch.bernoulli(self.pkeep* torch.ones(indices.shape, device= indices.device))\n",
    "        mask= mask.round().to(dtype= torch.int64)\n",
    "        # value: 0- 19; size: (B, 49).\n",
    "        random_indices= torch.randint_like(indices, 20)\n",
    "        new_indices= mask* indices+ (1- mask)* random_indices\n",
    "        # new_indices, (B, 50)\n",
    "        new_indices= torch.cat((sos_tokens, new_indices), dim= 1)\n",
    "        # (B, 49)\n",
    "        target= indices\n",
    "        # (B, 49, 20),\n",
    "        logits, _= self.transformer(new_indices[:, :-1])\n",
    "        # (B, 49, 20); (B, 49)\n",
    "        return logits, target\n",
    "\n",
    "    def top_k_logits(self, logits, k):\n",
    "        # logits, (B, 49, 20); k, 2.\n",
    "        # v, (B, 49, 2); ix, (B, 49, 2).\n",
    "        v, ix= torch.topk(logits, k)\n",
    "        out= logits.clone()\n",
    "        # mask on out, < top k.\n",
    "        out[out< v[:,[-1]]]= -float('inf')\n",
    "        return out\n",
    "\n",
    "    @torch.no_grad()\n",
    "    def sample(self, x, c, steps, temperature= 1.0, topk= 10):\n",
    "        # x, (B, 24); c, (B, 1); steps, 25.\n",
    "        self.transformer.eval()\n",
    "        # x, (B, 25).\n",
    "        x= torch.cat((c, x), dim= 1)\n",
    "        # for each step\n",
    "        for k in range(steps):\n",
    "            # x, (B, 25); logits, (B, 25, 20).\n",
    "            logits, _= self.transformer(x)\n",
    "            # (B, 1, 20)\n",
    "            logits= logits[:, -1, :]/ temperature\n",
    "            if topk is not None:\n",
    "                logits= self.top_k_logits(logits, topk)\n",
    "            # (B, 1, 20)\n",
    "            probs= F.softmax(logits, dim= -1)\n",
    "            # (B, 1)\n",
    "            ix= torch.multinomial(probs, num_samples= 1)\n",
    "            # (B, 26)\n",
    "            x= torch.cat((x, ix), dim= 1)\n",
    "        # (B, 49)\n",
    "        x= x[:, c.shape[1]:]\n",
    "        self.transformer.train()\n",
    "        return x\n",
    "\n",
    "    @torch.no_grad()\n",
    "    def log_images(self, x):\n",
    "        log= dict()\n",
    "        # (B, 49)\n",
    "        _, indices= self.encode_to_z(x)\n",
    "        # (B, 1)\n",
    "        sos_tokens= torch.ones(x.shape[0], 1)* self.sos_token\n",
    "        sos_tokens= sos_tokens.long().to('cuda')\n",
    "        # (B, 24)\n",
    "        start_indices= indices[:, :indices.shape[1]// 2]\n",
    "        # (B, 24); (B, 1); 49- 24= 25>> (B, 49)\n",
    "        sample_indices= self.sample(start_indices, sos_tokens, steps= indices.shape[1]- start_indices.shape[1])\n",
    "        # (B, 49)>> (B, 1, 28, 28)\n",
    "        half_sample= self.z_to_image(sample_indices)\n",
    "        start_indices= indices[:, :0]\n",
    "        sample_indices= self.sample(start_indices, sos_tokens, steps= indices.shape[1])\n",
    "        full_sample= self.z_to_image(sample_indices)\n",
    "        x_rec= self.z_to_image(indices)\n",
    "        log['input']= x\n",
    "        log['rec']= x_rec\n",
    "        log['half_sample']= half_sample\n",
    "        log['full_sample']= full_sample\n",
    "        return log, torch.stack((x[0].permute(1, 2, 0), x_rec, half_sample, full_sample))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "67ef072b-68f7-4e26-bc5f-78114659e96f",
   "metadata": {},
   "outputs": [],
   "source": [
    "tfm= Transformer(vqgan).to(device)\n",
    "opt_tf= torch.optim.Adam(tfm.parameters(), lr= lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "17c5f235-3c04-4411-b6cb-632ec7a68276",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, step: 1, loss: 3.3929390907287598\n",
      "epoch: 1, early stopping time/full time: 0/80, valid loss: 0.1882128305733204\n",
      "epoch: 2, step: 1, loss: 1.5431443452835083\n",
      "epoch: 2, early stopping time/full time: 0/80, valid loss: 0.18160506710410118\n",
      "epoch: 3, step: 1, loss: 1.4612029790878296\n",
      "epoch: 3, early stopping time/full time: 0/80, valid loss: 0.17802437022328377\n",
      "epoch: 4, step: 1, loss: 1.4525023698806763\n",
      "epoch: 4, early stopping time/full time: 0/80, valid loss: 0.1748204380273819\n",
      "epoch: 5, step: 1, loss: 1.4139416217803955\n",
      "epoch: 5, early stopping time/full time: 0/80, valid loss: 0.17368420585989952\n",
      "epoch: 6, step: 1, loss: 1.3782861232757568\n",
      "epoch: 6, early stopping time/full time: 0/80, valid loss: 0.1722600143402815\n",
      "epoch: 7, step: 1, loss: 1.3980712890625\n",
      "epoch: 7, early stopping time/full time: 0/80, valid loss: 0.1711117047816515\n",
      "epoch: 8, step: 1, loss: 1.371095895767212\n",
      "epoch: 8, early stopping time/full time: 0/80, valid loss: 0.17089124396443367\n",
      "epoch: 9, step: 1, loss: 1.3609049320220947\n",
      "epoch: 9, early stopping time/full time: 0/80, valid loss: 0.17056360468268394\n",
      "epoch: 10, step: 1, loss: 1.3538910150527954\n",
      "epoch: 10, early stopping time/full time: 0/80, valid loss: 0.16872519813477993\n",
      "epoch: 11, step: 1, loss: 1.3687127828598022\n",
      "epoch: 11, early stopping time/full time: 0/80, valid loss: 0.1686344724148512\n",
      "epoch: 12, step: 1, loss: 1.3836132287979126\n",
      "epoch: 12, early stopping time/full time: 0/80, valid loss: 0.16828379221260548\n",
      "epoch: 13, step: 1, loss: 1.391148328781128\n",
      "epoch: 13, early stopping time/full time: 1/80, valid loss: 0.16871045529842377\n",
      "epoch: 14, step: 1, loss: 1.341192603111267\n",
      "epoch: 14, early stopping time/full time: 0/80, valid loss: 0.16820457391440868\n",
      "epoch: 15, step: 1, loss: 1.3378922939300537\n",
      "epoch: 15, early stopping time/full time: 1/80, valid loss: 0.16879566945135593\n",
      "epoch: 16, step: 1, loss: 1.3056875467300415\n",
      "epoch: 16, early stopping time/full time: 0/80, valid loss: 0.1672355681657791\n",
      "epoch: 17, step: 1, loss: 1.3183099031448364\n",
      "epoch: 17, early stopping time/full time: 1/80, valid loss: 0.16729320958256721\n",
      "epoch: 18, step: 1, loss: 1.3293081521987915\n",
      "epoch: 18, early stopping time/full time: 0/80, valid loss: 0.16676022857427597\n",
      "epoch: 19, step: 1, loss: 1.3501662015914917\n",
      "epoch: 19, early stopping time/full time: 0/80, valid loss: 0.16598661616444588\n",
      "epoch: 20, step: 1, loss: 1.331997036933899\n",
      "epoch: 20, early stopping time/full time: 1/80, valid loss: 0.16674042493104935\n",
      "epoch: 21, step: 1, loss: 1.3561676740646362\n",
      "epoch: 21, early stopping time/full time: 2/80, valid loss: 0.1660072486847639\n",
      "epoch: 22, step: 1, loss: 1.3233180046081543\n",
      "epoch: 22, early stopping time/full time: 3/80, valid loss: 0.16636856645345688\n",
      "epoch: 23, step: 1, loss: 1.3121739625930786\n",
      "epoch: 23, early stopping time/full time: 4/80, valid loss: 0.16610775887966156\n",
      "epoch: 24, step: 1, loss: 1.3113298416137695\n",
      "epoch: 24, early stopping time/full time: 0/80, valid loss: 0.16547139175236225\n",
      "epoch: 25, step: 1, loss: 1.2749955654144287\n",
      "epoch: 25, early stopping time/full time: 1/80, valid loss: 0.165790481492877\n",
      "epoch: 26, step: 1, loss: 1.363341212272644\n",
      "epoch: 26, early stopping time/full time: 0/80, valid loss: 0.16537784039974213\n",
      "epoch: 27, step: 1, loss: 1.2981956005096436\n",
      "epoch: 27, early stopping time/full time: 0/80, valid loss: 0.16448214277625084\n",
      "epoch: 28, step: 1, loss: 1.3150895833969116\n",
      "epoch: 28, early stopping time/full time: 1/80, valid loss: 0.16535184159874916\n",
      "epoch: 29, step: 1, loss: 1.3541673421859741\n",
      "epoch: 29, early stopping time/full time: 2/80, valid loss: 0.1654357835650444\n",
      "epoch: 30, step: 1, loss: 1.2916078567504883\n",
      "epoch: 30, early stopping time/full time: 3/80, valid loss: 0.16525433398783207\n",
      "epoch: 31, step: 1, loss: 1.3473509550094604\n",
      "epoch: 31, early stopping time/full time: 4/80, valid loss: 0.1650355737656355\n",
      "epoch: 32, step: 1, loss: 1.3133363723754883\n",
      "epoch: 32, early stopping time/full time: 5/80, valid loss: 0.16494880244135857\n",
      "epoch: 33, step: 1, loss: 1.3393696546554565\n",
      "epoch: 33, early stopping time/full time: 0/80, valid loss: 0.16424578800797462\n",
      "epoch: 34, step: 1, loss: 1.3155848979949951\n",
      "epoch: 34, early stopping time/full time: 1/80, valid loss: 0.16442927718162537\n",
      "epoch: 35, step: 1, loss: 1.3414753675460815\n",
      "epoch: 35, early stopping time/full time: 2/80, valid loss: 0.16480349749326706\n",
      "epoch: 36, step: 1, loss: 1.3120514154434204\n",
      "epoch: 36, early stopping time/full time: 0/80, valid loss: 0.1639552339911461\n",
      "epoch: 37, step: 1, loss: 1.3057894706726074\n",
      "epoch: 37, early stopping time/full time: 1/80, valid loss: 0.16400250419974327\n",
      "epoch: 38, step: 1, loss: 1.337907314300537\n",
      "epoch: 38, early stopping time/full time: 2/80, valid loss: 0.16430416889488697\n",
      "epoch: 39, step: 1, loss: 1.3108371496200562\n",
      "epoch: 39, early stopping time/full time: 0/80, valid loss: 0.16372091695666313\n",
      "epoch: 40, step: 1, loss: 1.3401315212249756\n",
      "epoch: 40, early stopping time/full time: 1/80, valid loss: 0.164521349593997\n",
      "epoch: 41, step: 1, loss: 1.2846192121505737\n",
      "epoch: 41, early stopping time/full time: 2/80, valid loss: 0.1641307044774294\n",
      "epoch: 42, step: 1, loss: 1.3239136934280396\n",
      "epoch: 42, early stopping time/full time: 3/80, valid loss: 0.16427800059318542\n",
      "epoch: 43, step: 1, loss: 1.3118040561676025\n",
      "epoch: 43, early stopping time/full time: 4/80, valid loss: 0.16474383883178234\n",
      "epoch: 44, step: 1, loss: 1.3250118494033813\n",
      "epoch: 44, early stopping time/full time: 0/80, valid loss: 0.16311759874224663\n",
      "epoch: 45, step: 1, loss: 1.2907949686050415\n",
      "epoch: 45, early stopping time/full time: 1/80, valid loss: 0.16439385153353214\n",
      "epoch: 46, step: 1, loss: 1.305203914642334\n",
      "epoch: 46, early stopping time/full time: 2/80, valid loss: 0.16361596807837486\n",
      "epoch: 47, step: 1, loss: 1.3261834383010864\n",
      "epoch: 47, early stopping time/full time: 3/80, valid loss: 0.1634644977748394\n",
      "epoch: 48, step: 1, loss: 1.3323090076446533\n",
      "epoch: 48, early stopping time/full time: 0/80, valid loss: 0.1628650724887848\n",
      "epoch: 49, step: 1, loss: 1.327928066253662\n",
      "epoch: 49, early stopping time/full time: 1/80, valid loss: 0.16398137249052525\n",
      "epoch: 50, step: 1, loss: 1.3049650192260742\n",
      "epoch: 50, early stopping time/full time: 2/80, valid loss: 0.16350717842578888\n",
      "epoch: 51, step: 1, loss: 1.3249858617782593\n",
      "epoch: 51, early stopping time/full time: 3/80, valid loss: 0.16303144954144955\n",
      "epoch: 52, step: 1, loss: 1.2828384637832642\n",
      "epoch: 52, early stopping time/full time: 4/80, valid loss: 0.16352139227092266\n",
      "epoch: 53, step: 1, loss: 1.3576624393463135\n",
      "epoch: 53, early stopping time/full time: 5/80, valid loss: 0.16339914128184319\n",
      "epoch: 54, step: 1, loss: 1.2923376560211182\n",
      "epoch: 54, early stopping time/full time: 6/80, valid loss: 0.16359304636716843\n",
      "epoch: 55, step: 1, loss: 1.2824575901031494\n",
      "epoch: 55, early stopping time/full time: 0/80, valid loss: 0.16255096718668938\n",
      "epoch: 56, step: 1, loss: 1.3236349821090698\n",
      "epoch: 56, early stopping time/full time: 1/80, valid loss: 0.1638353206217289\n",
      "epoch: 57, step: 1, loss: 1.3145252466201782\n",
      "epoch: 57, early stopping time/full time: 2/80, valid loss: 0.16452892497181892\n",
      "epoch: 58, step: 1, loss: 1.3127729892730713\n",
      "epoch: 58, early stopping time/full time: 3/80, valid loss: 0.16348520293831825\n",
      "epoch: 59, step: 1, loss: 1.266658902168274\n",
      "epoch: 59, early stopping time/full time: 4/80, valid loss: 0.1641564927995205\n",
      "epoch: 60, step: 1, loss: 1.338997721672058\n",
      "epoch: 60, early stopping time/full time: 5/80, valid loss: 0.16346626356244087\n",
      "epoch: 61, step: 1, loss: 1.2978134155273438\n",
      "epoch: 61, early stopping time/full time: 6/80, valid loss: 0.16325371153652668\n",
      "epoch: 62, step: 1, loss: 1.2620875835418701\n",
      "epoch: 62, early stopping time/full time: 7/80, valid loss: 0.16341179236769676\n",
      "epoch: 63, step: 1, loss: 1.2701610326766968\n",
      "epoch: 63, early stopping time/full time: 8/80, valid loss: 0.16334765404462814\n",
      "epoch: 64, step: 1, loss: 1.265868902206421\n",
      "epoch: 64, early stopping time/full time: 9/80, valid loss: 0.16395217180252075\n",
      "epoch: 65, step: 1, loss: 1.2776165008544922\n",
      "epoch: 65, early stopping time/full time: 10/80, valid loss: 0.1633685752749443\n",
      "epoch: 66, step: 1, loss: 1.3104735612869263\n",
      "epoch: 66, early stopping time/full time: 11/80, valid loss: 0.16259255446493626\n",
      "epoch: 67, step: 1, loss: 1.282418131828308\n",
      "epoch: 67, early stopping time/full time: 12/80, valid loss: 0.1632234863936901\n",
      "epoch: 68, step: 1, loss: 1.296401023864746\n",
      "epoch: 68, early stopping time/full time: 13/80, valid loss: 0.163017887622118\n",
      "epoch: 69, step: 1, loss: 1.2716461420059204\n",
      "epoch: 69, early stopping time/full time: 0/80, valid loss: 0.16244886629283428\n",
      "epoch: 70, step: 1, loss: 1.3026992082595825\n",
      "epoch: 70, early stopping time/full time: 1/80, valid loss: 0.16318643651902676\n",
      "epoch: 71, step: 1, loss: 1.2828772068023682\n",
      "epoch: 71, early stopping time/full time: 2/80, valid loss: 0.1628445591777563\n",
      "epoch: 72, step: 1, loss: 1.309077262878418\n",
      "epoch: 72, early stopping time/full time: 3/80, valid loss: 0.16301747225224972\n",
      "epoch: 73, step: 1, loss: 1.2629766464233398\n",
      "epoch: 73, early stopping time/full time: 4/80, valid loss: 0.16278068162500858\n",
      "epoch: 74, step: 1, loss: 1.2906160354614258\n",
      "epoch: 74, early stopping time/full time: 0/80, valid loss: 0.1622460912913084\n",
      "epoch: 75, step: 1, loss: 1.2787619829177856\n",
      "epoch: 75, early stopping time/full time: 1/80, valid loss: 0.16273692809045315\n",
      "epoch: 76, step: 1, loss: 1.2887015342712402\n",
      "epoch: 76, early stopping time/full time: 2/80, valid loss: 0.1630358248949051\n",
      "epoch: 77, step: 1, loss: 1.261354684829712\n",
      "epoch: 77, early stopping time/full time: 3/80, valid loss: 0.1623467430472374\n",
      "epoch: 78, step: 1, loss: 1.257539987564087\n",
      "epoch: 78, early stopping time/full time: 4/80, valid loss: 0.16298515908420086\n",
      "epoch: 79, step: 1, loss: 1.30794358253479\n",
      "epoch: 79, early stopping time/full time: 5/80, valid loss: 0.16286431811749935\n",
      "epoch: 80, step: 1, loss: 1.300289511680603\n",
      "epoch: 80, early stopping time/full time: 6/80, valid loss: 0.1630306839942932\n",
      "epoch: 81, step: 1, loss: 1.3089479207992554\n",
      "epoch: 81, early stopping time/full time: 0/80, valid loss: 0.16191895864903927\n",
      "epoch: 82, step: 1, loss: 1.2855490446090698\n",
      "epoch: 82, early stopping time/full time: 1/80, valid loss: 0.16191796585917473\n",
      "epoch: 83, step: 1, loss: 1.327304482460022\n",
      "epoch: 83, early stopping time/full time: 2/80, valid loss: 0.1628076359629631\n",
      "epoch: 84, step: 1, loss: 1.3041019439697266\n",
      "epoch: 84, early stopping time/full time: 3/80, valid loss: 0.16206182911992073\n",
      "epoch: 85, step: 1, loss: 1.3451155424118042\n",
      "epoch: 85, early stopping time/full time: 4/80, valid loss: 0.1619691476225853\n",
      "epoch: 86, step: 1, loss: 1.299827218055725\n",
      "epoch: 86, early stopping time/full time: 0/80, valid loss: 0.1618859264999628\n",
      "epoch: 87, step: 1, loss: 1.300742506980896\n",
      "epoch: 87, early stopping time/full time: 1/80, valid loss: 0.16250665672123432\n",
      "epoch: 88, step: 1, loss: 1.320419192314148\n",
      "epoch: 88, early stopping time/full time: 2/80, valid loss: 0.1626031044870615\n",
      "epoch: 89, step: 1, loss: 1.29560387134552\n",
      "epoch: 89, early stopping time/full time: 3/80, valid loss: 0.1621789764612913\n",
      "epoch: 90, step: 1, loss: 1.31751549243927\n",
      "epoch: 90, early stopping time/full time: 0/80, valid loss: 0.16181096248328686\n",
      "epoch: 91, step: 1, loss: 1.2879009246826172\n",
      "epoch: 91, early stopping time/full time: 1/80, valid loss: 0.16310598701238632\n",
      "epoch: 92, step: 1, loss: 1.2827906608581543\n",
      "epoch: 92, early stopping time/full time: 2/80, valid loss: 0.16243943385779858\n",
      "epoch: 93, step: 1, loss: 1.3133457899093628\n",
      "epoch: 93, early stopping time/full time: 3/80, valid loss: 0.1622851137071848\n",
      "epoch: 94, step: 1, loss: 1.2816522121429443\n",
      "epoch: 94, early stopping time/full time: 0/80, valid loss: 0.16171006485819817\n",
      "epoch: 95, step: 1, loss: 1.3379898071289062\n",
      "epoch: 95, early stopping time/full time: 1/80, valid loss: 0.16219774447381496\n",
      "epoch: 96, step: 1, loss: 1.3310372829437256\n",
      "epoch: 96, early stopping time/full time: 2/80, valid loss: 0.16215365752577782\n",
      "epoch: 97, step: 1, loss: 1.2182754278182983\n",
      "epoch: 97, early stopping time/full time: 3/80, valid loss: 0.16235296986997128\n",
      "epoch: 98, step: 1, loss: 1.286064624786377\n",
      "epoch: 98, early stopping time/full time: 4/80, valid loss: 0.16315704584121704\n",
      "epoch: 99, step: 1, loss: 1.2895742654800415\n",
      "epoch: 99, early stopping time/full time: 5/80, valid loss: 0.16255983337759972\n",
      "epoch: 100, step: 1, loss: 1.2812144756317139\n",
      "epoch: 100, early stopping time/full time: 6/80, valid loss: 0.1630361806601286\n",
      "epoch: 101, step: 1, loss: 1.2743160724639893\n",
      "epoch: 101, early stopping time/full time: 7/80, valid loss: 0.16205372475087643\n",
      "epoch: 102, step: 1, loss: 1.3247121572494507\n",
      "epoch: 102, early stopping time/full time: 8/80, valid loss: 0.16193466074764729\n",
      "epoch: 103, step: 1, loss: 1.3196284770965576\n",
      "epoch: 103, early stopping time/full time: 9/80, valid loss: 0.16246244870126247\n",
      "epoch: 104, step: 1, loss: 1.2438857555389404\n",
      "epoch: 104, early stopping time/full time: 10/80, valid loss: 0.16238069348037243\n",
      "epoch: 105, step: 1, loss: 1.3007060289382935\n",
      "epoch: 105, early stopping time/full time: 0/80, valid loss: 0.16126220673322678\n",
      "epoch: 106, step: 1, loss: 1.27872896194458\n",
      "epoch: 106, early stopping time/full time: 1/80, valid loss: 0.16161981783807278\n",
      "epoch: 107, step: 1, loss: 1.3214905261993408\n",
      "epoch: 107, early stopping time/full time: 2/80, valid loss: 0.16174385882914066\n",
      "epoch: 108, step: 1, loss: 1.2694246768951416\n",
      "epoch: 108, early stopping time/full time: 3/80, valid loss: 0.1619432233273983\n",
      "epoch: 109, step: 1, loss: 1.3338978290557861\n",
      "epoch: 109, early stopping time/full time: 4/80, valid loss: 0.1628885194659233\n",
      "epoch: 110, step: 1, loss: 1.2594571113586426\n",
      "epoch: 110, early stopping time/full time: 5/80, valid loss: 0.16255594044923782\n",
      "epoch: 111, step: 1, loss: 1.3123315572738647\n",
      "epoch: 111, early stopping time/full time: 6/80, valid loss: 0.16140601225197315\n",
      "epoch: 112, step: 1, loss: 1.310415267944336\n",
      "epoch: 112, early stopping time/full time: 7/80, valid loss: 0.162617776542902\n",
      "epoch: 113, step: 1, loss: 1.318129062652588\n",
      "epoch: 113, early stopping time/full time: 8/80, valid loss: 0.16323604062199593\n",
      "epoch: 114, step: 1, loss: 1.3066986799240112\n",
      "epoch: 114, early stopping time/full time: 0/80, valid loss: 0.16120036877691746\n",
      "epoch: 115, step: 1, loss: 1.2717561721801758\n",
      "epoch: 115, early stopping time/full time: 1/80, valid loss: 0.16282016783952713\n",
      "epoch: 116, step: 1, loss: 1.296059012413025\n",
      "epoch: 116, early stopping time/full time: 2/80, valid loss: 0.16218717209994793\n",
      "epoch: 117, step: 1, loss: 1.3039487600326538\n",
      "epoch: 117, early stopping time/full time: 3/80, valid loss: 0.16202091984450817\n",
      "epoch: 118, step: 1, loss: 1.2747852802276611\n",
      "epoch: 118, early stopping time/full time: 4/80, valid loss: 0.1622252482920885\n",
      "epoch: 119, step: 1, loss: 1.2675275802612305\n",
      "epoch: 119, early stopping time/full time: 5/80, valid loss: 0.16122478432953358\n",
      "epoch: 120, step: 1, loss: 1.252274513244629\n",
      "epoch: 120, early stopping time/full time: 6/80, valid loss: 0.1613832488656044\n",
      "epoch: 121, step: 1, loss: 1.3091908693313599\n",
      "epoch: 121, early stopping time/full time: 0/80, valid loss: 0.16030628234148026\n",
      "epoch: 122, step: 1, loss: 1.253419041633606\n",
      "epoch: 122, early stopping time/full time: 1/80, valid loss: 0.1623117495328188\n",
      "epoch: 123, step: 1, loss: 1.2843595743179321\n",
      "epoch: 123, early stopping time/full time: 2/80, valid loss: 0.16209938935935497\n",
      "epoch: 124, step: 1, loss: 1.2774869203567505\n",
      "epoch: 124, early stopping time/full time: 3/80, valid loss: 0.1608808245509863\n",
      "epoch: 125, step: 1, loss: 1.2611984014511108\n",
      "epoch: 125, early stopping time/full time: 4/80, valid loss: 0.16199119947850704\n",
      "epoch: 126, step: 1, loss: 1.3113716840744019\n",
      "epoch: 126, early stopping time/full time: 5/80, valid loss: 0.16283699497580528\n",
      "epoch: 127, step: 1, loss: 1.278664231300354\n",
      "epoch: 127, early stopping time/full time: 6/80, valid loss: 0.16154902055859566\n",
      "epoch: 128, step: 1, loss: 1.2754255533218384\n",
      "epoch: 128, early stopping time/full time: 7/80, valid loss: 0.16142668016254902\n",
      "epoch: 129, step: 1, loss: 1.2914725542068481\n",
      "epoch: 129, early stopping time/full time: 8/80, valid loss: 0.16194142773747444\n",
      "epoch: 130, step: 1, loss: 1.299665927886963\n",
      "epoch: 130, early stopping time/full time: 9/80, valid loss: 0.16184351406991482\n",
      "epoch: 131, step: 1, loss: 1.3357232809066772\n",
      "epoch: 131, early stopping time/full time: 10/80, valid loss: 0.16170046851038933\n",
      "epoch: 132, step: 1, loss: 1.3217377662658691\n",
      "epoch: 132, early stopping time/full time: 11/80, valid loss: 0.16160716488957405\n",
      "epoch: 133, step: 1, loss: 1.2904952764511108\n",
      "epoch: 133, early stopping time/full time: 12/80, valid loss: 0.1617660205811262\n",
      "epoch: 134, step: 1, loss: 1.2984414100646973\n",
      "epoch: 134, early stopping time/full time: 13/80, valid loss: 0.16147588938474655\n",
      "epoch: 135, step: 1, loss: 1.2506961822509766\n",
      "epoch: 135, early stopping time/full time: 14/80, valid loss: 0.16202451847493649\n",
      "epoch: 136, step: 1, loss: 1.3251917362213135\n",
      "epoch: 136, early stopping time/full time: 15/80, valid loss: 0.16180905513465405\n",
      "epoch: 137, step: 1, loss: 1.2863460779190063\n",
      "epoch: 137, early stopping time/full time: 16/80, valid loss: 0.1622217558324337\n",
      "epoch: 138, step: 1, loss: 1.3085932731628418\n",
      "epoch: 138, early stopping time/full time: 17/80, valid loss: 0.16120858304202557\n",
      "epoch: 139, step: 1, loss: 1.2951107025146484\n",
      "epoch: 139, early stopping time/full time: 18/80, valid loss: 0.1618846356868744\n",
      "epoch: 140, step: 1, loss: 1.2858492136001587\n",
      "epoch: 140, early stopping time/full time: 19/80, valid loss: 0.16263780742883682\n",
      "epoch: 141, step: 1, loss: 1.305369257926941\n",
      "epoch: 141, early stopping time/full time: 20/80, valid loss: 0.16138207912445068\n",
      "epoch: 142, step: 1, loss: 1.2860941886901855\n",
      "epoch: 142, early stopping time/full time: 21/80, valid loss: 0.16123314388096333\n",
      "epoch: 143, step: 1, loss: 1.305817723274231\n",
      "epoch: 143, early stopping time/full time: 22/80, valid loss: 0.16234108060598373\n",
      "epoch: 144, step: 1, loss: 1.3094778060913086\n",
      "epoch: 144, early stopping time/full time: 23/80, valid loss: 0.16280335560441017\n",
      "epoch: 145, step: 1, loss: 1.3052403926849365\n",
      "epoch: 145, early stopping time/full time: 24/80, valid loss: 0.16217286698520184\n",
      "epoch: 146, step: 1, loss: 1.2879366874694824\n",
      "epoch: 146, early stopping time/full time: 25/80, valid loss: 0.16118705831468105\n",
      "epoch: 147, step: 1, loss: 1.2901533842086792\n",
      "epoch: 147, early stopping time/full time: 26/80, valid loss: 0.16124637238681316\n",
      "epoch: 148, step: 1, loss: 1.3211956024169922\n",
      "epoch: 148, early stopping time/full time: 27/80, valid loss: 0.16052493266761303\n",
      "epoch: 149, step: 1, loss: 1.320670485496521\n",
      "epoch: 149, early stopping time/full time: 28/80, valid loss: 0.16253365203738213\n",
      "epoch: 150, step: 1, loss: 1.3358967304229736\n",
      "epoch: 150, early stopping time/full time: 29/80, valid loss: 0.16125843487679958\n",
      "epoch: 151, step: 1, loss: 1.3048659563064575\n",
      "epoch: 151, early stopping time/full time: 30/80, valid loss: 0.16159430146217346\n",
      "epoch: 152, step: 1, loss: 1.3411527872085571\n",
      "epoch: 152, early stopping time/full time: 31/80, valid loss: 0.16151808388531208\n",
      "epoch: 153, step: 1, loss: 1.28170645236969\n",
      "epoch: 153, early stopping time/full time: 32/80, valid loss: 0.16099528595805168\n",
      "epoch: 154, step: 1, loss: 1.3013055324554443\n",
      "epoch: 154, early stopping time/full time: 33/80, valid loss: 0.16090145707130432\n",
      "epoch: 155, step: 1, loss: 1.3256765604019165\n",
      "epoch: 155, early stopping time/full time: 34/80, valid loss: 0.16176977567374706\n",
      "epoch: 156, step: 1, loss: 1.2760000228881836\n",
      "epoch: 156, early stopping time/full time: 35/80, valid loss: 0.16099688783288002\n",
      "epoch: 157, step: 1, loss: 1.2959296703338623\n",
      "epoch: 157, early stopping time/full time: 36/80, valid loss: 0.16188615001738071\n",
      "epoch: 158, step: 1, loss: 1.3216490745544434\n",
      "epoch: 158, early stopping time/full time: 37/80, valid loss: 0.16151374019682407\n",
      "epoch: 159, step: 1, loss: 1.2911359071731567\n",
      "epoch: 159, early stopping time/full time: 38/80, valid loss: 0.16180677711963654\n",
      "epoch: 160, step: 1, loss: 1.2897312641143799\n",
      "epoch: 160, early stopping time/full time: 39/80, valid loss: 0.16107618995010853\n",
      "epoch: 161, step: 1, loss: 1.3109465837478638\n",
      "epoch: 161, early stopping time/full time: 40/80, valid loss: 0.16187693551182747\n",
      "epoch: 162, step: 1, loss: 1.3091257810592651\n",
      "epoch: 162, early stopping time/full time: 41/80, valid loss: 0.16127088479697704\n",
      "epoch: 163, step: 1, loss: 1.2795634269714355\n",
      "epoch: 163, early stopping time/full time: 42/80, valid loss: 0.1606801114976406\n",
      "epoch: 164, step: 1, loss: 1.3034218549728394\n",
      "epoch: 164, early stopping time/full time: 43/80, valid loss: 0.1613392699509859\n",
      "epoch: 165, step: 1, loss: 1.3096215724945068\n",
      "epoch: 165, early stopping time/full time: 44/80, valid loss: 0.161500483751297\n",
      "epoch: 166, step: 1, loss: 1.3140194416046143\n",
      "epoch: 166, early stopping time/full time: 45/80, valid loss: 0.16217546164989471\n",
      "epoch: 167, step: 1, loss: 1.3004059791564941\n",
      "epoch: 167, early stopping time/full time: 46/80, valid loss: 0.16171014308929443\n",
      "epoch: 168, step: 1, loss: 1.2661962509155273\n",
      "epoch: 168, early stopping time/full time: 47/80, valid loss: 0.1614748276770115\n",
      "epoch: 169, step: 1, loss: 1.2920883893966675\n",
      "epoch: 169, early stopping time/full time: 48/80, valid loss: 0.1616997830569744\n",
      "epoch: 170, step: 1, loss: 1.3229018449783325\n",
      "epoch: 170, early stopping time/full time: 49/80, valid loss: 0.16202090866863728\n",
      "epoch: 171, step: 1, loss: 1.272910237312317\n",
      "epoch: 171, early stopping time/full time: 50/80, valid loss: 0.16079563833773136\n",
      "epoch: 172, step: 1, loss: 1.2751429080963135\n",
      "epoch: 172, early stopping time/full time: 51/80, valid loss: 0.16116825304925442\n",
      "epoch: 173, step: 1, loss: 1.2456912994384766\n",
      "epoch: 173, early stopping time/full time: 52/80, valid loss: 0.16191818937659264\n",
      "epoch: 174, step: 1, loss: 1.3050960302352905\n",
      "epoch: 174, early stopping time/full time: 53/80, valid loss: 0.16095528937876225\n",
      "epoch: 175, step: 1, loss: 1.2784271240234375\n",
      "epoch: 175, early stopping time/full time: 54/80, valid loss: 0.16193773597478867\n",
      "epoch: 176, step: 1, loss: 1.3222861289978027\n",
      "epoch: 176, early stopping time/full time: 55/80, valid loss: 0.16191444545984268\n",
      "epoch: 177, step: 1, loss: 1.3045244216918945\n",
      "epoch: 177, early stopping time/full time: 56/80, valid loss: 0.16216915100812912\n",
      "epoch: 178, step: 1, loss: 1.3108320236206055\n",
      "epoch: 178, early stopping time/full time: 57/80, valid loss: 0.16118504106998444\n",
      "epoch: 179, step: 1, loss: 1.257944107055664\n",
      "epoch: 179, early stopping time/full time: 58/80, valid loss: 0.1611049771308899\n",
      "epoch: 180, step: 1, loss: 1.3269586563110352\n",
      "epoch: 180, early stopping time/full time: 59/80, valid loss: 0.16180244088172913\n",
      "epoch: 181, step: 1, loss: 1.2851357460021973\n",
      "epoch: 181, early stopping time/full time: 60/80, valid loss: 0.16115578636527061\n",
      "epoch: 182, step: 1, loss: 1.2834346294403076\n",
      "epoch: 182, early stopping time/full time: 61/80, valid loss: 0.161332281306386\n",
      "epoch: 183, step: 1, loss: 1.2996962070465088\n",
      "epoch: 183, early stopping time/full time: 62/80, valid loss: 0.16192477196455002\n",
      "epoch: 184, step: 1, loss: 1.2592473030090332\n",
      "epoch: 184, early stopping time/full time: 63/80, valid loss: 0.1606261171400547\n",
      "epoch: 185, step: 1, loss: 1.3030015230178833\n",
      "epoch: 185, early stopping time/full time: 64/80, valid loss: 0.16118941083550453\n",
      "epoch: 186, step: 1, loss: 1.3098959922790527\n",
      "epoch: 186, early stopping time/full time: 65/80, valid loss: 0.16151718609035015\n",
      "epoch: 187, step: 1, loss: 1.2966691255569458\n",
      "epoch: 187, early stopping time/full time: 66/80, valid loss: 0.16118326596915722\n",
      "epoch: 188, step: 1, loss: 1.2805004119873047\n",
      "epoch: 188, early stopping time/full time: 67/80, valid loss: 0.1609430182725191\n",
      "epoch: 189, step: 1, loss: 1.2897230386734009\n",
      "epoch: 189, early stopping time/full time: 68/80, valid loss: 0.16098512895405293\n",
      "epoch: 190, step: 1, loss: 1.320132851600647\n",
      "epoch: 190, early stopping time/full time: 69/80, valid loss: 0.161405136808753\n",
      "epoch: 191, step: 1, loss: 1.2451483011245728\n",
      "epoch: 191, early stopping time/full time: 70/80, valid loss: 0.1607904341071844\n",
      "epoch: 192, step: 1, loss: 1.3188165426254272\n",
      "epoch: 192, early stopping time/full time: 71/80, valid loss: 0.16112411953508854\n",
      "epoch: 193, step: 1, loss: 1.3036476373672485\n",
      "epoch: 193, early stopping time/full time: 72/80, valid loss: 0.16136865690350533\n",
      "epoch: 194, step: 1, loss: 1.270470142364502\n",
      "epoch: 194, early stopping time/full time: 73/80, valid loss: 0.1610633209347725\n",
      "epoch: 195, step: 1, loss: 1.287136197090149\n",
      "epoch: 195, early stopping time/full time: 74/80, valid loss: 0.1608521994203329\n",
      "epoch: 196, step: 1, loss: 1.2698808908462524\n",
      "epoch: 196, early stopping time/full time: 75/80, valid loss: 0.16128708608448505\n",
      "epoch: 197, step: 1, loss: 1.2821550369262695\n",
      "epoch: 197, early stopping time/full time: 76/80, valid loss: 0.16072758845984936\n",
      "epoch: 198, step: 1, loss: 1.3167133331298828\n",
      "epoch: 198, early stopping time/full time: 77/80, valid loss: 0.16118582896888256\n",
      "epoch: 199, step: 1, loss: 1.2996752262115479\n",
      "epoch: 199, early stopping time/full time: 78/80, valid loss: 0.1611982397735119\n",
      "epoch: 200, step: 1, loss: 1.2941069602966309\n",
      "epoch: 200, early stopping time/full time: 79/80, valid loss: 0.16136611253023148\n",
      "epoch: 201, step: 1, loss: 1.2873566150665283\n",
      "epoch: 201, early stopping time/full time: 80/80, valid loss: 0.16084745712578297\n",
      "epoch: 202, step: 1, loss: 1.279746651649475\n"
     ]
    }
   ],
   "source": [
    "best_score, score, time, first= 0, 0, 0, True\n",
    "for epoch in range(epoch_num):\n",
    "    for step, (pic, label) in enumerate(train_loader):\n",
    "        pic= pic.view(pic.shape[0], 1, 28, 28).to(device)\n",
    "        logits, targets= tfm(pic)\n",
    "        loss= F.cross_entropy(logits.reshape(-1, logits.size(-1)), targets.reshape(-1))\n",
    "        opt_tf.zero_grad()\n",
    "        loss.backward()\n",
    "        opt_tf.step()\n",
    "        if step% 100== 0: print(f'epoch: {epoch+ 1}, step: {step+ 1}, loss: {loss}')\n",
    "    tfm.eval();val_loss= 0\n",
    "    for step, (pic, label) in enumerate(valid_loader):\n",
    "        pic= pic.view(pic.shape[0], 1, 28, 28).to(device)\n",
    "        logits, targets= tfm(pic)\n",
    "        loss= F.cross_entropy(logits.reshape(-1, logits.size(-1)), targets.reshape(-1))\n",
    "        val_loss+= loss.item()\n",
    "    val_loss= val_loss/ len(valid_loader)\n",
    "    # early stopping\n",
    "    if first==True or val_loss< best_score- 1e-4:\n",
    "        first= False\n",
    "        best_score= val_loss\n",
    "        time= 0\n",
    "        torch.save(tfm, f'{pt_folder}//{pt_file4tfm}')\n",
    "    elif time>= patience:\n",
    "        tfm= torch.load(f'{pt_folder}//{pt_file4tfm}')\n",
    "        break\n",
    "    else:\n",
    "        time+= 1\n",
    "    print(f'epoch: {epoch+ 1}, early stopping time/full time: {time}/{patience}, valid loss: {val_loss* 1.0/ len(valid_loader)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "bdb7074c-bda8-48ae-b8d8-edd8adf93e8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "tfm= torch.load(f'{pt_folder}//{pt_file4tfm}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "34ac9ddb-b3ba-4a0e-ab62-516cd375e305",
   "metadata": {},
   "outputs": [],
   "source": [
    "img= test_set[2][0].view(1, 1, 28, 28).cuda()\n",
    "log, sampled_imgs= tfm.log_images(img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "ffad2fad-53f8-43eb-b407-bd1698a3c68e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7oAAADjCAYAAACrbWjPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAATXklEQVR4nO3da2yWZ/0H8LulB1pomTA1doxE3UCUdNNg4jEaM3U659Spb+YLE3VhWUxm9IUnJp4SMXthjLipmVk0RmVhcczpYhaXLRUDc2rIVGCICuEQ3MG2QIGe/i/8v1nc7/fAXbrevfr5vP3mvq9rT3v16bf3eH5t09PT0xUAAAAUon2uNwAAAAAXkqILAABAURRdAAAAiqLoAgAAUBRFFwAAgKIougAAABRF0QUAAKAoii4AAABF6ah74dTUVHXkyJGqr6+vamtru5B7gmJMT09Xo6Oj1cDAQNXe3ry/KznHcG6cZZj/mn6Oq8pZhnNxrme5dtE9cuRIdemll9a9HBaUQ4cOVStXrpzrbfwP5xjOj7MM819Tz3FVOctwPlqd5dp/zurr66t7KSw4TT0vTd0XNFVTz0xT9wVN1OTz0uS9QdO0Oi+1i67/nQLOXVPPS1P3BU3V1DPT1H1BEzX5vDR5b9A0rc5LM/+BAgAAANSk6AIAAFAURRcAAICiKLoAAAAURdEFAACgKIouAAAARVF0AQAAKIqiCwAAQFEUXQAAAIqi6AIAAFCUjrneAAAAAOeup6cnzMbGxp7HnTSXJ7oAAAAURdEFAACgKIouAAAARVF0AQAAKIqiCwAAQFEUXQAAAIqi6AIAAFAUc3QBAAAC7e35s8Hu7u4w6+3tDbOOjriKrVu3Ll1z7969YXb06NEwm5ycTO9bEk90AQAAKIqiCwAAQFEUXQAAAIqi6AIAAFAURRcAAICiKLoAAAAUxXghAABgQctGCL3//e9Pr73lllvCLBs9tGHDhjAbGhpK1zxz5kya44kuAAAAhVF0AQAAKIqiCwAAQFEUXQAAAIqi6AIAAFAURRcAAICiGC8EAAAUb/HixWF21VVXhdldd92V3rezszPM7r777jD74x//mN6XmfFEFwAAgKIougAAABRF0QUAAKAoii4AAABFUXQBAAAoiqILAABAUYwXAorR1tYWZtPT0xf8nlWVjxRob4//lpiNODhz5ky65tTUVJidPXs2vTZS9/WB2eAsO8sQecELXpDmt9xyS5hdffXVYfbKV74yzJYuXZqu+fe//z3MPvOZz6TXMns80QUAAKAoii4AAABFUXQBAAAoiqILAABAURRdAAAAiqLoAgAAUBRFFwAAgKIsmDm6b3rTm9L8ox/9aJiNjY2F2Y4dO8Ls9OnT6ZrXXHNNmH3sYx8Ls9makbdv374wGxoaCrPs9amqqtqyZUuY7dmzp/XG4P/19vam+erVq8PsFa94RZitX78+zFatWlV7zRUrVoRZNpez1c+ORYsWhdmxY8fC7MCBA2G2ffv2dM0HHnggzIaHh8PMTE+ei7P8X84yPLfXvOY1YZb9TlpV+czs7LxmJiYm0nz37t1h9uSTT9Zak5nzRBcAAICiKLoAAAAURdEFAACgKIouAAAARVF0AQAAKIqiCwAAQFHapmt+XvzIyEi1bNmyC72fGRkYGAiz+++/P732iiuuCLPZ+kj9bOzAyZMnw2zv3r1hlo1HmIm+vr4w6+7uTq89ceJEmK1ZsybMstEK883w8HDV398/19v4H3N1jrORGu9973vD7Nvf/nZ632wESEdHPE1tcnIyzFqNIsjybBxBNsZjdHQ0XTP7Xurq6gqzbK/ZOa2qqtq5c2eYZePZRkZG0vvON87ysznLzvJ81NRzXFXN/P16tmRjgH74wx+GWfZ92kp2lqempsIs22tVVdXjjz8eZq973evC7OzZs+l9ybU6y57oAgAAUBRFFwAAgKIougAAABRF0QUAAKAoii4AAABFUXQBAAAoSvwZ/fPQbbfdFmaDg4PP407+66tf/Wqa/+IXvwizP//5zxd2MzN05ZVXhtn27dvTay+55JIwe8tb3hJmP//5z1vui+bKPor/1a9+dZj94Ac/CLPly5fXXjMbG3DmzJkw++c//5muuWPHjjD78Y9/HGbZeLEnn3wyXfOpp54Ks56enjDLzvFXvvKVdM3Xv/71YXb55ZeH2WOPPZbel+abT2d5fHw8zFqd5UcffTTM7rjjjjA7fvx4mJ06dSpd8z//+U+Y1T3Lt956a7pmdu2qVavCLBufAnVko8Ky98jsPbuVbGRo9nOn1XiybEzQkiVLal3HzHmiCwAAQFEUXQAAAIqi6AIAAFAURRcAAICiKLoAAAAURdEFAACgKI0cL5R93PjWrVvD7Nprr6295smTJ8Ns8+bNYXb77beH2TPPPJOumY1IaJps3NGJEydq3/fBBx+sfS3N1tXVFWbZ6K1ly5aFWTY6pKryUR0PPfRQmGXneOfOnema2ZiDbIzBbMnO49DQUK3rqqqqOjs7w2wu/jt5/syns/y9730vzH7/+9+nazbtLGejiR5++OEw27VrV3rfyy67LMxWrFjRemNwgfT394fZW9/61jBbtGhR7TWz372zUT+tRoVlZ+cDH/hAmN11111hNjk5ma5Ja57oAgAAUBRFFwAAgKIougAAABRF0QUAAKAoii4AAABFUXQBAAAoiqILAABAURo5R/dLX/pSmF133XW17vnLX/4yzb/whS+E2eOPP15rzZIMDg6G2aWXXvo87oT54u1vf3uYrV+/PsyyGXf33HNPumZ2jo8ePRpmY2Nj6X1LsXLlyjBbs2ZNem32Gh08eLD2nmg+Z7l5stnG2ft1VeUzSBfK60fzLV26NMw6OvL6ks29zr7Hs1nzx48fT9d82cteFmY33nhjrTX37t2brklrnugCAABQFEUXAACAoii6AAAAFEXRBQAAoCiKLgAAAEVRdAEAAChKI8cLTU5OhtmOHTvC7Lvf/W6Y/fSnP53Rnha6r3/962G2ZMmS9Nrvf//7YfbUU0/V3hNzq709/zvZihUrwuzAgQNhtmvXrjD79Kc/na559uzZNC9FW1tbmHV3d4fZl7/85TDr7e1N1/ztb38bZs8880x6Lc3mLM8/2QihK664Ir12fHw8zLIRKtn3STZKCiLLli0Ls/7+/jDLxgdVVVWNjIyE2X333RdmW7ZsCbPDhw+nay5fvjzMNm3aFGY33HBDmGXjVquq9euAJ7oAAAAURtEFAACgKIouAAAARVF0AQAAKIqiCwAAQFEUXQAAAIrSNl3zs6lHRkbSjwVn/lm8eHGYnTp1KsyycVBVVVXXXnttmD3wwAOtN1aA4eHh9KPy58psnuOenp4wy8bjjI2NhZmP0v+vrq6uMHvjG98YZtu3bw+z06dPp2t+8IMfDLNHHnkkzEr7mjnLz+Ysz55snM/WrVvD7D3veU9632zc4sc//vEwa/VeP5809RxX1cL6/fqFL3xhmP3ud78Ls1Zfu2yU3kMPPRRm+/fvD7NWP7P6+vrC7CMf+UiYLV26NMy++c1vpmsa69X6LHuiCwAAQFEUXQAAAIqi6AIAAFAURRcAAICiKLoAAAAURdEFAACgKB1zvQGa484776x1XTbmoKoWzgghni0bLcLMLFq0KMxuvPHGMMvGEv3mN79J19y1a1eYGRVTNmd5bnR3d4dZNkas1RigTZs21b4WLqSnn346zD7xiU+E2cqVK9P7Dg0NhdmxY8fCbGJiIsyy992qqqqLLroozJYvXx5m9957b5gZHzRznugCAABQFEUXAACAoii6AAAAFEXRBQAAoCiKLgAAAEVRdAEAACiKogsAAEBRzNFdYC677LIwu/7662vdc/PmzXW3A9Tw2te+Nsze8Y53hNnw8HCYff7zn0/XPH36dOuNARfM4sWLa2X79+9P73v48OHae4ILKZvbnM3C7ejI60s28zabTdvW1hZmAwMD6Zpf/OIXw2x8fDzM9u3bl96XmfFEFwAAgKIougAAABRF0QUAAKAoii4AAABFUXQBAAAoiqILAABAUYwXWmDWrl0bZp2dnWG2bdu2MHviiSdmtCfg2bLRCFVVVTfddFOt+/7kJz8JswMHDqTXTk9P11oTqOfFL35xmGU/I372s5+l952YmKi9J3i+ZKOHsqyqqqq7uzvMlixZEmb9/f1htmHDhnTNbOTRt771rTAzum92eaILAABAURRdAAAAiqLoAgAAUBRFFwAAgKIougAAABRF0QUAAKAoxgsVZsWKFWn+ox/9KMza2trC7EMf+lDtPQHnp7e3N81XrVoVZvv27Quzb3zjG2HWalxDNs4kGz00NTWV3hcWsmwMyte+9rUwy0YE3XfffTPaE8x34+PjYZa9l33uc58Ls4svvjhdc8uWLWH2t7/9LcyM7ptdnugCAABQFEUXAACAoii6AAAAFEXRBQAAoCiKLgAAAEVRdAEAACiK8ULzUHt7/PeJd73rXem1/f39YZZ9xPlsfTT6P/7xjzC75pprat8Xmq6jI/7x+853vjO99pJLLgmzgwcPhtng4GCYtRoD9O53vzvMdu7cGWbZqJOxsbF0TSjd2972tjC76qqrwuzQoUNhlo0Yg4Ugez87ffp0mA0MDITZiRMn0jV37NgRZq3G9zF7PNEFAACgKIouAAAARVF0AQAAKIqiCwAAQFEUXQAAAIqi6AIAAFAURRcAAICimKPbUCtWrAiz73znO2H24Q9/eDa2U61ZsybMZjJHdybXQhNkc617e3vD7A1veEOYbdy4MV3zRS96UZh1dnaG2W233RZm2Yztqqqqtra2MFu9enWYPfLII2Fmji6l6+npSfNbb701zLq7u8Ps/vvvD7Px8fHWG4MFKpuxu2fPnjBrNd/e77PN5IkuAAAARVF0AQAAKIqiCwAAQFEUXQAAAIqi6AIAAFAURRcAAICiGC/UUL/61a/CbP369bOy5oMPPlgr27ZtW3rf4eHhMJucnGy9MZhD2Vidqqqqjo74x+j1118fZjfddFOYXX755emaXV1dYbZ06dIwO3ToUJjt378/XfMPf/hDmN19991hduzYsfS+ULKrr746za+88sowO3nyZJjdfvvtYWbMCdSTvV9lY/2qqqoGBgbC7F//+lftPTEznugCAABQFEUXAACAoii6AAAAFEXRBQAAoCiKLgAAAEVRdAEAACiK8UINdc8994TZpk2bwuyzn/1set9169aF2c033xxmrUaPQKlajerIRmRNTEyEWTY6ZHR0NF2zt7c3zA4cOBBmN9xwQ5j9+9//Ttc8c+ZMrQxK194ePzPYuHFjem02Kuwvf/lLmBlXAhfeqVOnwqynpye99s1vfnOYOa9zxxNdAAAAiqLoAgAAUBRFFwAAgKIougAAABRF0QUAAKAoii4AAABFMV6ooTZv3hxm73vf+8Is+3jzqqqqPXv2hJkRQnD+svFC27dvD7OXv/zltbKqqqq2trYwu/POO8MsG3GQ/XdUVesxS7BQrV27Nsxe9apXpddm40yycWBAPR0dcfW57rrrwqy7uzu979GjR2vvidnjiS4AAABFUXQBAAAoiqILAABAURRdAAAAiqLoAgAAUBRFFwAAgKIougAAABTFHN15aPXq1WHWatbl7t27L/R2gMDJkyfD7Iknngiz/v7+9L4nTpwIs1//+tdhNjExkd4XeG7Z7Oo77rgjzDo7O9P7Pvzww2GW/YwA6snm4V588cVh9te//jW976OPPlp7T8weT3QBAAAoiqILAABAURRdAAAAiqLoAgAAUBRFFwAAgKIougAAABTFeKEF5rHHHpvrLcCCkY37WrduXZi1Gkly/PjxMDt27FjrjQHnZenSpWE2ODgYZmNjY+l9P/nJT4bZ1NRU640B5+UlL3lJmE1OTobZhg0b0vuOjo7W3hOzxxNdAAAAiqLoAgAAUBRFFwAAgKIougAAABRF0QUAAKAoii4AAABFMV4IYA6sXbs2zLq6utJrn3766TAbHx+vvSfgub30pS8Ns8WLF4fZrl270vvu37+/9p6A85eN/Lr55pvD7E9/+lN632ycIHPHE10AAACKougCAABQFEUXAACAoii6AAAAFEXRBQAAoCiKLgAAAEUxXmge2r17d5gNDQ2l127btu1CbwcIZOMG7r333jAbHBxM77tx48YwO3v2bOuNAefl6NGjYbZ169Yw+9SnPpXed3JysvaegPOXneXDhw8/jzvh+eCJLgAAAEVRdAEAACiKogsAAEBRFF0AAACKougCAABQFEUXAACAoii6AAAAFKVtOhv0mBgZGamWLVt2ofcDRRoeHq76+/vnehv/wzmeXW1tbWGWve4XXXRRet+DBw+G2dTUVMt9UZ+zvDB1dHSEWXbOJyYm0vvW/BWMGWrqOa4qZxnOR6uz7IkuAAAARVF0AQAAKIqiCwAAQFEUXQAAAIqi6AIAAFCU+GMEW/BJgXDumnpemrqvUmSvb5a1+uRkX7e509TXvqn7KkXd19fXpZma/HVp8t6gaVqdl9pFd3R0tO6lsOCMjo42clyAczx3hoeHa2XMLWd5YZqcnJzrLXABNfUcV5WzDOej1VmuPUd3amqqOnLkSNXX15fOkIOFbHp6uhodHa0GBgaq9vbm/UsB5xjOjbMM81/Tz3FVOctwLs71LNcuugAAANBEzfxzFgAAANSk6AIAAFAURRcAAICiKLoAAAAURdEFAACgKIouAAAARVF0AQAAKIqiCwAAQFEUXQAAAIqi6AIAAFAURRcAAICiKLoAAAAU5f8AARlm3oSDC7UAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 1200x1200 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "imgs= [img.detach().cpu().squeeze() for img in sampled_imgs]\n",
    "show_mnist(imgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8f17225-992a-479e-b31a-6ab36197df2d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
